fix(glue): support Ray jobs with Runtime parameter (#25867)

AWS Glue recently changed API and CloudFormation spec to require `Runtime` parameter for Ray jobs. To address the changes, I am submitting this PR. This PR is for fixing the issue #25787. ## Reference CloudFormation doc has been also updated for `Runtime` parameter. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-job-jobcommand.html#cfn-glue-job-jobcommand-runtime ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Jun 29, 2023 · 8153237 · 8153237
1 parent af913b3
commit 8153237
Show file tree

Hide file tree

Showing 12 changed files with 163 additions and 100 deletions.
diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md
@@ -94,6 +94,7 @@ new glue.Job(this, 'RayJob', {
   executable: glue.JobExecutable.pythonRay({
     glueVersion: glue.GlueVersion.V4_0,
     pythonVersion: glue.PythonVersion.THREE_NINE,
+    runtime: glue.Runtime.RAY_TWO_FOUR,
     script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),
   }),
   workerType: glue.WorkerType.Z_2X,

diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts
@@ -87,6 +87,34 @@ export enum PythonVersion {
   THREE_NINE = '3.9',
 }
 
+/**
+ * AWS Glue runtime determines the runtime engine of the job.
+ *
+ */
+export class Runtime {
+  /**
+   * Runtime for a Glue for Ray 2.4.
+   */
+  public static readonly RAY_TWO_FOUR = new Runtime('Ray2.4');
+
+  /**
+   * Custom runtime
+   * @param runtime custom runtime
+   */
+  public static of(runtime: string): Runtime {
+    return new Runtime(runtime);
+  }
+
+  /**
+   * The name of this Runtime.
+   */
+  public readonly name: string;
+
+  private constructor(name: string) {
+    this.name = name;
+  }
+}
+
 /**
  * The job type.
  *
@@ -150,6 +178,12 @@ interface PythonExecutableProps {
 }
 
 interface SharedJobExecutableProps {
+  /**
+   * Runtime. It is required for Ray jobs.
+   *
+   */
+  readonly runtime?: Runtime;
+
   /**
    * Glue version.
    *
@@ -347,6 +381,9 @@ export class JobExecutable {
     if (config.pythonVersion === PythonVersion.THREE && config.type === JobType.RAY) {
       throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray');
     }
+    if (config.runtime === undefined && config.type === JobType.RAY) {
+      throw new Error('Runtime is required for Ray jobs.');
+    }
     this.config = config;
   }
 
@@ -388,6 +425,13 @@ export interface JobExecutableConfig {
    */
   readonly pythonVersion?: PythonVersion;
 
+  /**
+   * The Runtime to use.
+   *
+   * @default - no runtime specified
+   */
+  readonly runtime?: Runtime;
+
   /**
    * The script that is executed by a job.
    */

diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts
@@ -685,6 +685,7 @@ export class Job extends JobBase {
         name: executable.type.name,
         scriptLocation: this.codeS3ObjectUrl(executable.script),
         pythonVersion: executable.pythonVersion,
+        runtime: executable.runtime ? executable.runtime.name : undefined,
       },
       glueVersion: executable.glueVersion.name,
       workerType: props.workerType?.name,

diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json
@@ -1,5 +1,5 @@
 {
-  "version": "29.0.0",
+  "version": "32.0.0",
   "files": {
     "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": {
       "source": {
@@ -14,15 +14,15 @@
         }
       }
     },
-    "b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce": {
+    "e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89": {
       "source": {
         "path": "aws-glue-job.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce.json",
+          "objectKey": "e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json
@@ -354,7 +354,7 @@
     "Tags": {
      "key": "value"
     },
-    "WorkerType": "G.025X"
+    "WorkerType": "G.1X"
    }
   },
   "EtlJob30ServiceRole8E675579": {
@@ -1415,6 +1415,7 @@
     "Command": {
      "Name": "glueray",
      "PythonVersion": "3.9",
+     "Runtime": "Ray2.4",
      "ScriptLocation": {
       "Fn::Join": [
        "",

diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out
@@ -1 +1 @@
-{"version":"29.0.0"}
+{"version":"32.0.0"}
diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json
@@ -1,5 +1,5 @@
 {
-  "version": "29.0.0",
+  "version": "32.0.0",
   "testCases": {
     "integ.job": {
       "stacks": [

diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json
@@ -1,5 +1,5 @@
 {
-  "version": "29.0.0",
+  "version": "32.0.0",
   "artifacts": {
     "aws-glue-job.assets": {
       "type": "cdk:asset-manifest",
@@ -17,7 +17,7 @@
         "validateOnSynth": false,
         "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
         "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
-        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/b553fef631f82898c826f3c20e1de0d155dbd3a35339ef92d0893052a5be69ce.json",
+        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json",
         "requiresBootstrapStackVersion": 6,
         "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
         "additionalDependencies": [