-
Notifications
You must be signed in to change notification settings - Fork 3.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(glue-alpha): adding partition projection #27490
Closed
Closed
Changes from all commits
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
dde68dd
Adding Partition Projection
guyernest aca83ff
Merge branch 'main' into main
guyernest 7082ecc
Merge branch 'main' into main
guyernest 5922652
Adding integration test for partition projection
guyernest 070ac51
Merge branch 'main' of https://github.com/guyernest/aws-cdk
guyernest 8374d0c
Merge branch 'main' into main
guyernest 42a72a9
Cleaning tests from unused variable
guyernest 26a18eb
Merge branch 'main' into partition-projection
guyernest c822b1b
Merge branch 'main' into partition-projection
guyernest eff2a4e
Merge branch 'main' into partition-projection
guyernest 31f63bd
Merge branch 'main' into partition-projection
guyernest 625df89
Merge branch 'main' into partition-projection
guyernest 1e34a63
Linting
guyernest e63eaa6
Adding docstring to public elements
guyernest File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
294 changes: 294 additions & 0 deletions
294
packages/@aws-cdk/aws-glue-alpha/lib/partition-projection.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,294 @@ | ||
/** | ||
* The partition projection type. | ||
* | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html#partition-projection-date-type | ||
*/ | ||
export enum PartitionProjectionType { | ||
/** | ||
* ENUM_TYPE | ||
*/ | ||
ENUM_TYPE = 'enum', | ||
/** | ||
* INTEGER_TYPE | ||
*/ | ||
INTEGER_TYPE = 'integer', | ||
/** | ||
* DATE_TYPE | ||
*/ | ||
DATE_TYPE = 'date', | ||
/** | ||
* INJECTED_TYPE | ||
*/ | ||
INJECTED_TYPE = 'injected', | ||
} | ||
|
||
/** | ||
* Dynamic Partition Projection Class | ||
* | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html | ||
* | ||
*/ | ||
export abstract class PartitionProjection { | ||
constructor( | ||
/** | ||
* the type of the partition projection | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-setting-up.html#partition-projection-specifying-custom-s3-storage-locations | ||
*/ | ||
public readonly type: PartitionProjectionType, | ||
/** | ||
* Required. The projection use for column columnName. | ||
*/ | ||
public readonly columnName: string, | ||
/** | ||
* The prefix format of the S3 bucket and keys that store the partitions. | ||
*/ | ||
public readonly storageLocationTemplate: string) {} | ||
|
||
/** | ||
* Get the parameter key for the partition projection | ||
* @param paramName the name of the parameter | ||
* @returns the parameter key for the partition projection | ||
*/ | ||
public getParameterKey(paramName: string): string { | ||
return `${this.columnName}.${paramName}`; | ||
} | ||
/** | ||
* Create the output format for the partition projection | ||
* @returns the output format for the partition projection | ||
*/ | ||
public toOutputFormat(): any { | ||
throw new Error('Method not implemented.'); | ||
} | ||
} | ||
|
||
/** | ||
* A time unit word that represents the serialized form of a ChronoUnit. | ||
* Possible values are YEARS, MONTHS, WEEKS, DAYS, HOURS, MINUTES, SECONDS, or MILLISECONDS. These values are case insensitive. | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html#partition-projection-date-type | ||
*/ | ||
export enum DateIntervalUnit { | ||
/** | ||
* YEARS | ||
*/ | ||
YEARS = 'YEARS', | ||
/** | ||
* MONTHS | ||
*/ | ||
MONTHS = 'MONTHS', | ||
/** | ||
* WEEKS | ||
*/ | ||
WEEKS = 'WEEKS', | ||
/** | ||
* DAYS | ||
*/ | ||
DAYS = 'DAYS', | ||
/** | ||
* HOURS | ||
*/ | ||
HOURS = 'HOURS', | ||
/** | ||
* MINUTES | ||
*/ | ||
MINUTES = 'MINUTES', | ||
/** | ||
* SECONDS | ||
*/ | ||
SECONDS = 'SECONDS', | ||
/** | ||
* MILLISECONDS | ||
*/ | ||
MILLISECONDS = 'MILLISECONDS', | ||
} | ||
|
||
/** | ||
* Implementation of DatePartitionProjection | ||
* | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html#partition-projection-date-type | ||
* @public | ||
*/ | ||
export class DatePartitionProjection extends PartitionProjection { | ||
/** | ||
* @param columnName | ||
* @param storageLocationTemplate | ||
* @param range | ||
* @param format | ||
* @param interval | ||
* @param intervalUnit | ||
*/ | ||
constructor( | ||
/** | ||
* Required. The projection use for column columnName. | ||
*/ | ||
public readonly columnName: string, | ||
/** | ||
* The prefix format of the S3 bucket and keys that store the partitions. | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-setting-up.html#partition-projection-specifying-custom-s3-storage-locations | ||
*/ | ||
public readonly storageLocationTemplate: string, | ||
/** | ||
* Required. A two-element, comma-separated list which provides the minimum and maximum range values for the column columnName. These values are inclusive and can use any format compatible with the Java java.time.* date types. Both the minimum and maximum values must use the same format. The format specified in the .format property must be the format used for these values. | ||
* | ||
* This column can also contain relative date strings, formatted in this regular expression pattern: | ||
* | ||
* \s*NOW\s*(([\+\-])\s*([0-9]+)\s*(YEARS?|MONTHS?|WEEKS?|DAYS?|HOURS?|MINUTES?|SECONDS?)\s*)? | ||
* | ||
* White spaces are allowed, but in date literals are considered part of the date strings themselves. | ||
*/ | ||
public readonly range: string, | ||
/** | ||
* Required. A date format string based on the Java date format DateTimeFormatter. Can be any supported Java.time.* type. | ||
*/ | ||
public readonly format: string, | ||
/** | ||
* A positive integer that specifies the interval between successive partition values for column columnName. For example, a range value of 2017-01,2018-12 with an interval value of 1 and an interval.unit value of MONTHS produces the values 2017-01, 2017-02, 2017-03, and so on. The same range value with an interval value of 2 and an interval.unit value of MONTHS produces the values 2017-01, 2017-03, 2017-05, and so on. Leading and trailing white space is allowed. | ||
* | ||
* When the provided dates are at single-day or single-month precision, the interval is optional and defaults to 1 day or 1 month, respectively. Otherwise, interval is required. | ||
*/ | ||
public readonly interval?: number, | ||
/** | ||
* A time unit word that represents the serialized form of a ChronoUnit. Possible values are YEARS, MONTHS, WEEKS, DAYS, HOURS, MINUTES, SECONDS, or MILLISECONDS. These values are case insensitive. | ||
*/ | ||
public readonly intervalUnit?: DateIntervalUnit, | ||
) { | ||
super( | ||
PartitionProjectionType.DATE_TYPE, | ||
columnName, | ||
storageLocationTemplate); | ||
} | ||
|
||
/** | ||
* Create the output format for the partition projection | ||
* @returns the output format for the partition projection | ||
*/ | ||
toOutputFormat(): any { | ||
const baseKey = `projection.${this.columnName}`; | ||
return { | ||
['projection.enabled']: true, | ||
['storage.location.template']: this.storageLocationTemplate, | ||
[`${baseKey}.type`]: this.type, | ||
[`${baseKey}.format`]: this.format, | ||
[`${baseKey}.range`]: this.range, | ||
[`${baseKey}.interval`]: this.interval ? String(this.interval) : undefined, | ||
[`${baseKey}.interval.unit`]: this.intervalUnit ? this.intervalUnit : undefined, | ||
}; | ||
} | ||
} | ||
|
||
/** | ||
* Implementation of IntegerPartitionProjection | ||
* | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html#partition-projection-integer-type | ||
*/ | ||
export class IntegerPartitionProjection extends PartitionProjection { | ||
constructor( | ||
/** | ||
* Required. The projection use for column columnName. | ||
*/ | ||
public readonly columnName: string, | ||
/** | ||
* The prefix format of the S3 bucket and keys that store the partitions. | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-setting-up.html#partition-projection-specifying-custom-s3-storage-locations | ||
*/ | ||
public readonly storageLocationTemplate: string, | ||
/** | ||
* Required. A two-element comma-separated list that provides the minimum and maximum range values to be returned by queries on the column columnName. Note that the values must be separated by a comma, not a hyphen. These values are inclusive, can be negative, and can have leading zeroes. Leading and trailing white space is allowed. | ||
*/ | ||
public readonly range: string, | ||
/** | ||
* Optional. A positive integer that specifies the interval between successive partition values for the column columnName. For example, a range value of "1,3" with an interval value of "1" produces the values 1, 2, and 3. The same range value with an interval value of "2" produces the values 1 and 3, skipping 2. Leading and trailing white space is allowed. The default is 1. | ||
*/ | ||
public readonly interval?: number, | ||
/** | ||
* Optional. A positive integer that specifies the number of digits to include in the partition value's final representation for column columnName. For example, a range value of "1,3" that has a digits value of "1" produces the values 1, 2, and 3. The same range value with a digits value of "2" produces the values 01, 02, and 03. Leading and trailing white space is allowed. The default is no static number of digits and no leading zeroes. | ||
*/ | ||
public readonly digits?: number) { | ||
super( | ||
PartitionProjectionType.INTEGER_TYPE, | ||
columnName, | ||
storageLocationTemplate); | ||
} | ||
|
||
/** | ||
* Create the output format for the partition projection | ||
* @returns the output format for the partition projection | ||
*/ | ||
toOutputFormat(): any { | ||
const baseKey = `projection.${this.columnName}`; | ||
return { | ||
['projection.enabled']: true, | ||
['storage.location.template']: this.storageLocationTemplate, | ||
[`${baseKey}.type`]: this.type, | ||
[`${baseKey}.range`]: this.range, | ||
[`${baseKey}.interval`]: this.interval ? String(this.interval) : undefined, | ||
[`${baseKey}.digits`]: this.digits ? String(this.digits) : undefined, | ||
}; | ||
} | ||
|
||
} | ||
/** | ||
* Implenetation of EnumPartitionProjection | ||
* | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html#partition-projection-enum-type | ||
*/ | ||
export class EnumPartitionProjection extends PartitionProjection { | ||
constructor( | ||
/** | ||
* Required. The projection use for column columnName. | ||
*/ | ||
public readonly columnName: string, | ||
/** | ||
* The prefix format of the S3 bucket and keys that store the partitions. | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-setting-up.html#partition-projection-specifying-custom-s3-storage-locations | ||
*/ | ||
public readonly storageLocationTemplate: string, | ||
/** | ||
* Required. A comma-separated list of enumerated partition values for column columnName. Any white space is considered part of an enum value. | ||
*/ | ||
public readonly values: string) { | ||
super( | ||
PartitionProjectionType.ENUM_TYPE, | ||
columnName, | ||
storageLocationTemplate, | ||
); | ||
} | ||
|
||
/** | ||
* Create the output format for the partition projection | ||
* @returns the output format for the partition projection | ||
*/ | ||
toOutputFormat(): any { | ||
const baseKey = `projection.${this.columnName}`; | ||
return { | ||
['projection.enabled']: true, | ||
['storage.location.template']: this.storageLocationTemplate, | ||
[`${baseKey}.type`]: this.type, | ||
[`${baseKey}.values`]: this.values, | ||
}; | ||
} | ||
|
||
} | ||
|
||
/** | ||
* Implementation of InjectedPartitionProjection | ||
* | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html#partition-projection-injected-type | ||
*/ | ||
export class InjectedPartitionProjection extends PartitionProjection { | ||
constructor( | ||
/** | ||
* Required. The projection use for column columnName. | ||
*/ | ||
public readonly columnName: string, | ||
/** | ||
* The prefix format of the S3 bucket and keys that store the partitions. | ||
* | ||
* @see https://docs.aws.amazon.com/athena/latest/ug/partition-projection-setting-up.html#partition-projection-specifying-custom-s3-storage-locations | ||
*/ | ||
public readonly storageLocationTemplate: string) { | ||
super( | ||
PartitionProjectionType.INJECTED_TYPE, | ||
columnName, | ||
storageLocationTemplate); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hey, @guyernest I see the following error,
I am curious, are you seeing these fail in PR builds or local builds too?