Fixed Failed Kokoro Test (Dataproc) [(#1203)](GoogleCloudPlatform/pyt…

…hon-docs-samples#1203) * Fixed Failed Kokoro Test (Dataproc) * Fixed Lint Error * Update dataproc_e2e_test.py * Update dataproc_e2e_test.py * Fixing More Lint Errors * Fixed b/65407087 * Revert "Merge branch 'master' of https://github.com/michaelawyu/python-docs-samples" This reverts commit 1614c7d3ef33630a8ab095792b27fc25fd91f0ad, reversing changes made to cd1dbfd25997a154a8a85cc754cc2a85b18a63c4. * Revert "Fixed b/65407087" This reverts commit cd1dbfd25997a154a8a85cc754cc2a85b18a63c4. * Fixed Lint Error * Fixed Lint Error
googleapis · Nov 8, 2017 · 4eb7dae · 4eb7dae
1 parent 9e21f7e
commit 4eb7dae
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 6 deletions.
diff --git a/samples/snippets/dataproc_e2e_test.py b/samples/snippets/dataproc_e2e_test.py
@@ -18,17 +18,14 @@
 
 import os
 
-from gcp_devrel.testing.flaky import flaky
-
 import submit_job_to_cluster
 
 PROJECT = os.environ['GCLOUD_PROJECT']
 BUCKET = os.environ['CLOUD_STORAGE_BUCKET']
-CLUSTER_NAME = 'testcluster2'
+CLUSTER_NAME = 'testcluster3'
 ZONE = 'us-central1-b'
 
 
-@flaky
 def test_e2e():
     output = submit_job_to_cluster.main(
         PROJECT, ZONE, CLUSTER_NAME, BUCKET)

diff --git a/samples/snippets/submit_job_to_cluster.py b/samples/snippets/submit_job_to_cluster.py
@@ -25,12 +25,12 @@
 def get_default_pyspark_file():
     """Gets the PySpark file from this directory"""
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    f = open(os.path.join(current_dir, DEFAULT_FILENAME), 'r')
+    f = open(os.path.join(current_dir, DEFAULT_FILENAME), 'rb')
     return f, DEFAULT_FILENAME
 
 
 def get_pyspark_file(filename):
-    f = open(filename, 'r')
+    f = open(filename, 'rb')
     return f, os.path.basename(filename)
 
 
@@ -76,6 +76,14 @@ def create_cluster(dataproc, project, zone, region, cluster_name):
         'config': {
             'gceClusterConfig': {
                 'zoneUri': zone_uri
+            },
+            'masterConfig': {
+                'numInstances': 1,
+                'machineTypeUri': 'n1-standard-1'
+            },
+            'workerConfig': {
+                'numInstances': 2,
+                'machineTypeUri': 'n1-standard-1'
             }
         }
     }