feat(client-firehose): This release 1) Add configurable buffering hin…

…ts for Snowflake as destination. 2) Add ReadFromTimestamp for MSK As Source. Firehose will start reading data from MSK Cluster using offset associated with this timestamp. 3) Gated public beta release to add Apache Iceberg tables as destination.
aws · Jul 18, 2024 · 9a36c7e · 9a36c7e
1 parent 3918268
commit 9a36c7e
Show file tree

Hide file tree

Showing 7 changed files with 863 additions and 17 deletions.
diff --git a/clients/client-firehose/src/auth/httpAuthSchemeProvider.ts b/clients/client-firehose/src/auth/httpAuthSchemeProvider.ts
@@ -60,7 +60,7 @@ function createAwsAuthSigv4HttpAuthOption(authParameters: FirehoseHttpAuthScheme
       name: "firehose",
       region: authParameters.region,
     },
-    propertiesExtractor: (config: FirehoseClientConfig, context) => ({
+    propertiesExtractor: (config: Partial<FirehoseClientConfig>, context) => ({
       /**
        * @internal
        */

diff --git a/clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts b/clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts
@@ -540,6 +540,7 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       RoleARN: "STRING_VALUE", // required
  *       Connectivity: "PUBLIC" || "PRIVATE", // required
  *     },
+ *     ReadFromTimestamp: new Date("TIMESTAMP"),
  *   },
  *   SnowflakeDestinationConfiguration: { // SnowflakeDestinationConfiguration
  *     AccountUrl: "STRING_VALUE", // required
@@ -572,6 +573,34 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       RoleARN: "STRING_VALUE",
  *       Enabled: true || false, // required
  *     },
+ *     BufferingHints: { // SnowflakeBufferingHints
+ *       SizeInMBs: Number("int"),
+ *       IntervalInSeconds: Number("int"),
+ *     },
+ *   },
+ *   IcebergDestinationConfiguration: { // IcebergDestinationConfiguration
+ *     DestinationTableConfigurationList: [ // DestinationTableConfigurationList
+ *       { // DestinationTableConfiguration
+ *         DestinationTableName: "STRING_VALUE", // required
+ *         DestinationDatabaseName: "STRING_VALUE", // required
+ *         UniqueKeys: [
+ *           "STRING_VALUE",
+ *         ],
+ *         S3ErrorOutputPrefix: "STRING_VALUE",
+ *       },
+ *     ],
+ *     BufferingHints: "<BufferingHints>",
+ *     CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ *     ProcessingConfiguration: "<ProcessingConfiguration>",
+ *     S3BackupMode: "FailedDataOnly" || "AllData",
+ *     RetryOptions: {
+ *       DurationInSeconds: Number("int"),
+ *     },
+ *     RoleARN: "STRING_VALUE", // required
+ *     CatalogConfiguration: { // CatalogConfiguration
+ *       CatalogARN: "STRING_VALUE",
+ *     },
+ *     S3Configuration: "<S3DestinationConfiguration>", // required
  *   },
  * };
  * const command = new CreateDeliveryStreamCommand(input);

diff --git a/clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts b/clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts
@@ -88,6 +88,7 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //           Connectivity: "PUBLIC" || "PRIVATE", // required
  * //         },
  * //         DeliveryStartTimestamp: new Date("TIMESTAMP"),
+ * //         ReadFromTimestamp: new Date("TIMESTAMP"),
  * //       },
  * //     },
  * //     Destinations: [ // DestinationDescriptionList // required
@@ -508,6 +509,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             RoleARN: "STRING_VALUE",
  * //             Enabled: true || false, // required
  * //           },
+ * //           BufferingHints: { // SnowflakeBufferingHints
+ * //             SizeInMBs: Number("int"),
+ * //             IntervalInSeconds: Number("int"),
+ * //           },
  * //         },
  * //         AmazonOpenSearchServerlessDestinationDescription: { // AmazonOpenSearchServerlessDestinationDescription
  * //           RoleARN: "STRING_VALUE",
@@ -535,6 +540,30 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             VpcId: "STRING_VALUE", // required
  * //           },
  * //         },
+ * //         IcebergDestinationDescription: { // IcebergDestinationDescription
+ * //           DestinationTableConfigurationList: [ // DestinationTableConfigurationList
+ * //             { // DestinationTableConfiguration
+ * //               DestinationTableName: "STRING_VALUE", // required
+ * //               DestinationDatabaseName: "STRING_VALUE", // required
+ * //               UniqueKeys: [
+ * //                 "STRING_VALUE",
+ * //               ],
+ * //               S3ErrorOutputPrefix: "STRING_VALUE",
+ * //             },
+ * //           ],
+ * //           BufferingHints: "<BufferingHints>",
+ * //           CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ * //           ProcessingConfiguration: "<ProcessingConfiguration>",
+ * //           S3BackupMode: "FailedDataOnly" || "AllData",
+ * //           RetryOptions: {
+ * //             DurationInSeconds: Number("int"),
+ * //           },
+ * //           RoleARN: "STRING_VALUE",
+ * //           CatalogConfiguration: { // CatalogConfiguration
+ * //             CatalogARN: "STRING_VALUE",
+ * //           },
+ * //           S3DestinationDescription: "<S3DestinationDescription>",
+ * //         },
  * //       },
  * //     ],
  * //     HasMoreDestinations: true || false, // required

diff --git a/clients/client-firehose/src/commands/UpdateDestinationCommand.ts b/clients/client-firehose/src/commands/UpdateDestinationCommand.ts
@@ -474,6 +474,43 @@ export interface UpdateDestinationCommandOutput extends UpdateDestinationOutput,
  *       RoleARN: "STRING_VALUE",
  *       Enabled: true || false, // required
  *     },
+ *     BufferingHints: { // SnowflakeBufferingHints
+ *       SizeInMBs: Number("int"),
+ *       IntervalInSeconds: Number("int"),
+ *     },
+ *   },
+ *   IcebergDestinationUpdate: { // IcebergDestinationUpdate
+ *     DestinationTableConfigurationList: [ // DestinationTableConfigurationList
+ *       { // DestinationTableConfiguration
+ *         DestinationTableName: "STRING_VALUE", // required
+ *         DestinationDatabaseName: "STRING_VALUE", // required
+ *         UniqueKeys: [
+ *           "STRING_VALUE",
+ *         ],
+ *         S3ErrorOutputPrefix: "STRING_VALUE",
+ *       },
+ *     ],
+ *     BufferingHints: "<BufferingHints>",
+ *     CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ *     ProcessingConfiguration: "<ProcessingConfiguration>",
+ *     S3BackupMode: "FailedDataOnly" || "AllData",
+ *     RetryOptions: {
+ *       DurationInSeconds: Number("int"),
+ *     },
+ *     RoleARN: "STRING_VALUE",
+ *     CatalogConfiguration: { // CatalogConfiguration
+ *       CatalogARN: "STRING_VALUE",
+ *     },
+ *     S3Configuration: { // S3DestinationConfiguration
+ *       RoleARN: "STRING_VALUE", // required
+ *       BucketARN: "STRING_VALUE", // required
+ *       Prefix: "STRING_VALUE",
+ *       ErrorOutputPrefix: "STRING_VALUE",
+ *       BufferingHints: "<BufferingHints>",
+ *       CompressionFormat: "UNCOMPRESSED" || "GZIP" || "ZIP" || "Snappy" || "HADOOP_SNAPPY",
+ *       EncryptionConfiguration: "<EncryptionConfiguration>",
+ *       CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
+ *     },
  *   },
  * };
  * const command = new UpdateDestinationCommand(input);