From 4c7fe9cf598f7a919ec76114c5b548ceb814fa14 Mon Sep 17 00:00:00 2001 From: George Fu Date: Tue, 31 Oct 2023 16:48:08 -0400 Subject: [PATCH] feat(lib-dynamodb): large number handling (#5427) * feat(lib-dynamodb): large number handling * feat(lib-dynamodb): large number handling docs and set test case * feat(lib-dynamodb): set release tag * feat(lib-dynamodb): remove unsafe conversion feature * feat(lib-dynamodb): add 1e100 number test case * feat(lib-dynamodb): large number handling, remove extra unmarshall option --- lib/lib-dynamodb/README.md | 131 ++++++++++++++---- lib/lib-dynamodb/src/index.ts | 2 + .../src/test/lib-dynamodb.e2e.spec.ts | 57 +++++--- .../util-dynamodb/src/NumberValue.spec.ts | 38 +++++ packages/util-dynamodb/src/NumberValue.ts | 88 ++++++++++++ .../util-dynamodb/src/convertToAttr.spec.ts | 22 +++ packages/util-dynamodb/src/convertToAttr.ts | 10 +- packages/util-dynamodb/src/convertToNative.ts | 10 +- packages/util-dynamodb/src/index.ts | 1 + packages/util-dynamodb/src/unmarshall.ts | 1 + 10 files changed, 314 insertions(+), 46 deletions(-) create mode 100644 packages/util-dynamodb/src/NumberValue.spec.ts create mode 100644 packages/util-dynamodb/src/NumberValue.ts diff --git a/lib/lib-dynamodb/README.md b/lib/lib-dynamodb/README.md index e821136fa2ba..72339396282f 100644 --- a/lib/lib-dynamodb/README.md +++ b/lib/lib-dynamodb/README.md @@ -20,18 +20,18 @@ Responses from DynamoDB are unmarshalled into plain JavaScript objects by the `DocumentClient`. The `DocumentClient` does not accept `AttributeValue`s in favor of native JavaScript types. -| JavaScript Type | DynamoDB AttributeValue | -| :-------------------------------: | ----------------------- | -| String | S | -| Number / BigInt | N | -| Boolean | BOOL | -| null | NULL | -| Array | L | -| Object | M | -| Set\ | BS | -| Set\ | NS | -| Set\ | SS | -| Uint8Array, Buffer, File, Blob... | B | +| JavaScript Type | DynamoDB AttributeValue | +| :--------------------------------: | ----------------------- | +| String | S | +| Number / BigInt / NumberValue | N | +| Boolean | BOOL | +| null | NULL | +| Array | L | +| Object | M | +| Set\ | BS | +| Set\ | NS | +| Set\ | SS | +| Uint8Array, Buffer, File, Blob... | B | ### Example @@ -98,20 +98,48 @@ const ddbDocClient = DynamoDBDocument.from(client); // client is DynamoDB client The configuration for marshalling and unmarshalling can be sent as an optional second parameter during creation of document client as follows: -```js -const marshallOptions = { - // Whether to automatically convert empty strings, blobs, and sets to `null`. - convertEmptyValues: false, // false, by default. - // Whether to remove undefined values while marshalling. - removeUndefinedValues: false, // false, by default. - // Whether to convert typeof object to map attribute. - convertClassInstanceToMap: false, // false, by default. -}; - -const unmarshallOptions = { - // Whether to return numbers as a string instead of converting them to native JavaScript numbers. - wrapNumbers: false, // false, by default. -}; +```ts +export interface marshallOptions { + /** + * Whether to automatically convert empty strings, blobs, and sets to `null` + */ + convertEmptyValues?: boolean; + /** + * Whether to remove undefined values while marshalling. + */ + removeUndefinedValues?: boolean; + /** + * Whether to convert typeof object to map attribute. + */ + convertClassInstanceToMap?: boolean; + /** + * Whether to convert the top level container + * if it is a map or list. + * + * Default is true when using the DynamoDBDocumentClient, + * but false if directly using the marshall function (backwards compatibility). + */ + convertTopLevelContainer?: boolean; +} + +export interface unmarshallOptions { + /** + * Whether to return numbers as a string instead of converting them to native JavaScript numbers. + * This allows for the safe round-trip transport of numbers of arbitrary size. + */ + wrapNumbers?: boolean; + + /** + * When true, skip wrapping the data in `{ M: data }` before converting. + * + * Default is true when using the DynamoDBDocumentClient, + * but false if directly using the unmarshall function (backwards compatibility). + */ + convertWithoutMapWrapper?: boolean; +} + +const marshallOptions: marshallOptions = {}; +const unmarshallOptions: unmarshallOptions = {}; const translateConfig = { marshallOptions, unmarshallOptions }; @@ -160,6 +188,57 @@ await ddbDocClient.put({ }); ``` +### Large Numbers and `NumberValue`. + +On the input or marshalling side, the class `NumberValue` can be used +anywhere to represent a DynamoDB number value, even small numbers. + +```ts +import { DynamoDB } from "@aws-sdk/client-dynamodb"; +import { NumberValue, DynamoDBDocument } from "@aws-sdk/lib-dynamodb"; + +// Note, the client will not validate the acceptability of the number +// in terms of size or format. +// It is only here to preserve your precise representation. +const client = DynamoDBDocument.from(new DynamoDB({})); + +await client.put({ + Item: { + id: 1, + smallNumber: NumberValue.from("123"), + bigNumber: NumberValue.from("1000000000000000000000.000000000001"), + nSet: new Set([123, NumberValue.from("456"), 789]), + }, +}); +``` + +On the output or unmarshalling side, the class `NumberValue` is used +depending on your setting for the `unmarshallOptions` flag `wrapnumbers`, +shown above. + +```ts +import { DynamoDB } from "@aws-sdk/client-dynamodb"; +import { NumberValue, DynamoDBDocument } from "@aws-sdk/lib-dynamodb"; + +const client = DynamoDBDocument.from(new DynamoDB({})); + +const response = await client.get({ + Key: { + id: 1, + }, +}); + +/** + * Numbers in the response may be a number, a BigInt, or a NumberValue depending + * on how you set `wrapNumbers`. + */ +const value = response.Item.bigNumber; +``` + +`NumberValue` does not provide a way to do mathematical operations on itself. +To do mathematical operations, take the string value of `NumberValue` by calling +`.toString()` and supply it to your chosen big number implementation. + ### Client and Command middleware stacks As with other AWS SDK for JavaScript v3 clients, you can apply middleware functions diff --git a/lib/lib-dynamodb/src/index.ts b/lib/lib-dynamodb/src/index.ts index c0a00ae48911..8b782ea2e99c 100644 --- a/lib/lib-dynamodb/src/index.ts +++ b/lib/lib-dynamodb/src/index.ts @@ -4,3 +4,5 @@ export * from "./DynamoDBDocumentClient"; // smithy-typescript generated code export * from "./commands"; export * from "./pagination"; + +export { NumberValueImpl as NumberValue } from "@aws-sdk/util-dynamodb"; diff --git a/lib/lib-dynamodb/src/test/lib-dynamodb.e2e.spec.ts b/lib/lib-dynamodb/src/test/lib-dynamodb.e2e.spec.ts index 4d24311c89b7..a6a5792a57b9 100644 --- a/lib/lib-dynamodb/src/test/lib-dynamodb.e2e.spec.ts +++ b/lib/lib-dynamodb/src/test/lib-dynamodb.e2e.spec.ts @@ -15,6 +15,7 @@ import { ExecuteStatementCommandOutput, ExecuteTransactionCommandOutput, GetCommandOutput, + NumberValue, PutCommandOutput, QueryCommandOutput, ScanCommandOutput, @@ -32,6 +33,9 @@ describe(DynamoDBDocument.name, () => { marshallOptions: { convertTopLevelContainer: true, }, + unmarshallOptions: { + wrapNumbers: true, + }, }); function throwIfError(e: unknown) { @@ -76,18 +80,27 @@ describe(DynamoDBDocument.name, () => { const data = { null: null, string: "myString", - number: 1, + number: NumberValue.from(1), + bigInt: NumberValue.from( + "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + ), + bigNumber: NumberValue.from("3210000000000000000.0000000000000123"), boolean: true, sSet: new Set(["my", "string", "set"]), - nSet: new Set([2, 3, 4]), + nSet: new Set([2, 3, 4].map(NumberValue.from)), list: [ null, "myString", - 1, + NumberValue.from(1), true, new Set(["my", "string", "set"]), - new Set([2, 3, 4]), - ["listInList", 1, null], + new Set([NumberValue.from(2), NumberValue.from(3), NumberValue.from(4)]), + new Set([ + NumberValue.from("3210000000000000000.0000000000000123"), + NumberValue.from("3210000000000000001.0000000000000123"), + NumberValue.from("3210000000000000002.0000000000000123"), + ]), + ["listInList", NumberValue.from(1), null], { mapInList: "mapInList", }, @@ -95,11 +108,11 @@ describe(DynamoDBDocument.name, () => { map: { null: null, string: "myString", - number: 1, + number: NumberValue.from(1), boolean: true, sSet: new Set(["my", "string", "set"]), - nSet: new Set([2, 3, 4]), - listInMap: ["listInMap", 1, null], + nSet: new Set([2, 3, 4].map(NumberValue.from)), + listInMap: ["listInMap", NumberValue.from(1), null], mapInMap: { mapInMap: "mapInMap" }, }, }; @@ -116,6 +129,9 @@ describe(DynamoDBDocument.name, () => { if (input instanceof Set) { return new Set([...input].map(updateTransform)) as T; } + if (input instanceof NumberValue) { + return NumberValue.from(input.toString()) as T; + } return Object.entries(input).reduce((acc, [k, v]) => { acc[updateTransform(k)] = updateTransform(v); return acc; @@ -436,28 +452,37 @@ describe(DynamoDBDocument.name, () => { expect(updateTransform(data)).toEqual({ "null-x": null, "string-x": "myString-x", - "number-x": 2, + "number-x": NumberValue.from(1), + "bigInt-x": NumberValue.from( + "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + ), + "bigNumber-x": NumberValue.from("3210000000000000000.0000000000000123"), "boolean-x": false, "sSet-x": new Set(["my-x", "string-x", "set-x"]), - "nSet-x": new Set([3, 4, 5]), + "nSet-x": new Set([2, 3, 4].map(NumberValue.from)), "list-x": [ null, "myString-x", - 2, + NumberValue.from(1), false, new Set(["my-x", "string-x", "set-x"]), - new Set([3, 4, 5]), - ["listInList-x", 2, null], + new Set([2, 3, 4].map(NumberValue.from)), + new Set([ + NumberValue.from("3210000000000000000.0000000000000123"), + NumberValue.from("3210000000000000001.0000000000000123"), + NumberValue.from("3210000000000000002.0000000000000123"), + ]), + ["listInList-x", NumberValue.from(1), null], { "mapInList-x": "mapInList-x" }, ], "map-x": { "null-x": null, "string-x": "myString-x", - "number-x": 2, + "number-x": NumberValue.from(1), "boolean-x": false, "sSet-x": new Set(["my-x", "string-x", "set-x"]), - "nSet-x": new Set([3, 4, 5]), - "listInMap-x": ["listInMap-x", 2, null], + "nSet-x": new Set([2, 3, 4].map(NumberValue.from)), + "listInMap-x": ["listInMap-x", NumberValue.from(1), null], "mapInMap-x": { "mapInMap-x": "mapInMap-x" }, }, }); diff --git a/packages/util-dynamodb/src/NumberValue.spec.ts b/packages/util-dynamodb/src/NumberValue.spec.ts new file mode 100644 index 000000000000..c7e3b047d1aa --- /dev/null +++ b/packages/util-dynamodb/src/NumberValue.spec.ts @@ -0,0 +1,38 @@ +import { NumberValue } from "./NumberValue"; + +const BIG_DECIMAL = + "123456789012345678901234567890123456789012345678901234567890.123456789012345678901234567890123456789012345678901234567890"; +const BIG_INT = "123456789012345678901234567890123456789012345678901234567890"; + +describe(NumberValue.name, () => { + it("can be statically constructed from numbers", () => { + expect(NumberValue.from(123.123).toString()).toEqual("123.123"); + + expect(() => NumberValue.from(1.23e100)).toThrow(); + expect(() => NumberValue.from(Infinity)).toThrow(); + expect(() => NumberValue.from(-Infinity)).toThrow(); + expect(() => NumberValue.from(NaN)).toThrow(); + }); + + it("can be statically constructed from strings", () => { + expect(NumberValue.from(BIG_DECIMAL).toString()).toEqual(BIG_DECIMAL); + }); + + it("can be statically constructed from BigInts", () => { + expect(NumberValue.from(BigInt(BIG_INT)).toString()).toEqual(BIG_INT); + }); + + it("can convert to AttributeValue", () => { + expect(NumberValue.from(BIG_DECIMAL).toAttributeValue()).toEqual({ + N: BIG_DECIMAL, + }); + }); + + it("can convert to string", () => { + expect(NumberValue.from(BIG_DECIMAL).toString()).toEqual(BIG_DECIMAL); + }); + + it("can convert to BigInt", () => { + expect(NumberValue.from(BIG_INT).toBigInt()).toEqual(BigInt(BIG_INT)); + }); +}); diff --git a/packages/util-dynamodb/src/NumberValue.ts b/packages/util-dynamodb/src/NumberValue.ts new file mode 100644 index 000000000000..d5cbe6f5aabb --- /dev/null +++ b/packages/util-dynamodb/src/NumberValue.ts @@ -0,0 +1,88 @@ +import { NumberValue as INumberValue } from "./models"; + +/** + * + * Class for storing DynamoDB numbers that exceed the scale of + * JavaScript's MAX_SAFE_INTEGER and MIN_SAFE_INTEGER, or the + * decimal precision limit. + * + * This class does not support mathematical operations in JavaScript. + * Convert the contained string value to your application-specific + * large number implementation to perform mathematical operations. + * + * @public + * + */ +export class NumberValue implements INumberValue { + public value: string; + + /** + * This class does not validate that your string input is a valid number. + * + * @param value - a precise number, or any BigInt or string, or AttributeValue. + */ + public constructor(value: number | Number | BigInt | string | { N: string }) { + if (typeof value === "object" && "N" in value) { + this.value = String(value.N); + } else { + this.value = String(value); + } + + const valueOf = typeof value.valueOf() === "number" ? (value.valueOf() as number) : 0; + const imprecise = + valueOf > Number.MAX_SAFE_INTEGER || + valueOf < Number.MIN_SAFE_INTEGER || + Math.abs(valueOf) === Infinity || + Number.isNaN(valueOf); + + if (imprecise) { + throw new Error( + `NumberValue should not be initialized with an imprecise number=${valueOf}. Use a string instead.` + ); + } + } + + /** + * This class does not validate that your string input is a valid number. + * + * @param value - a precise number, or any BigInt or string, or AttributeValue. + */ + public static from(value: number | Number | BigInt | string | { N: string }) { + return new NumberValue(value); + } + + /** + * @returns the AttributeValue form for DynamoDB. + */ + public toAttributeValue() { + return { + N: this.toString(), + }; + } + + /** + * @returns BigInt representation. + * + * @throws SyntaxError if the string representation is not convertable to a BigInt. + */ + public toBigInt() { + const stringValue = this.toString(); + return BigInt(stringValue); + } + + /** + * @override + * + * @returns string representation. This is the canonical format in DynamoDB. + */ + public toString() { + return String(this.value); + } + + /** + * @override + */ + public valueOf() { + return this.toString(); + } +} diff --git a/packages/util-dynamodb/src/convertToAttr.spec.ts b/packages/util-dynamodb/src/convertToAttr.spec.ts index 26be9d6aab11..07e77655522f 100644 --- a/packages/util-dynamodb/src/convertToAttr.spec.ts +++ b/packages/util-dynamodb/src/convertToAttr.spec.ts @@ -6,6 +6,7 @@ import { AttributeValue } from "@aws-sdk/client-dynamodb"; import { convertToAttr } from "./convertToAttr"; import { marshallOptions } from "./marshall"; import { NativeAttributeValue } from "./models"; +import { NumberValue } from "./NumberValue"; describe("convertToAttr", () => { describe("null", () => { @@ -115,6 +116,27 @@ describe("convertToAttr", () => { }); }); + describe("NumberValue", () => { + [true, false].forEach((convertClassInstanceToMap) => { + const maxSafe = BigInt(Number.MAX_SAFE_INTEGER); + [ + // @ts-expect-error BigInt literals are not available when targeting lower than ES2020. + 1n, + // @ts-expect-error BigInt literals are not available when targeting lower than ES2020. + maxSafe * 2n, + // @ts-expect-error BigInt literals are not available when targeting lower than ES2020. + maxSafe * -2n, + BigInt(Number.MAX_VALUE), + BigInt("0x1fffffffffffff"), + BigInt("0b11111111111111111111111111111111111111111111111111111"), + ].forEach((num) => { + it(`returns for NumberValue: ${num}`, () => { + expect(convertToAttr(NumberValue.from(num), { convertClassInstanceToMap })).toEqual({ N: num.toString() }); + }); + }); + }); + }); + describe("binary", () => { [true, false].forEach((convertClassInstanceToMap) => { const buffer = new ArrayBuffer(64); diff --git a/packages/util-dynamodb/src/convertToAttr.ts b/packages/util-dynamodb/src/convertToAttr.ts index d73527401b0e..e7e26c8bc45f 100644 --- a/packages/util-dynamodb/src/convertToAttr.ts +++ b/packages/util-dynamodb/src/convertToAttr.ts @@ -2,6 +2,7 @@ import { AttributeValue } from "@aws-sdk/client-dynamodb"; import { marshallOptions } from "./marshall"; import { NativeAttributeBinary, NativeAttributeValue, NativeScalarAttributeValue } from "./models"; +import { NumberValue } from "./NumberValue"; /** * Convert a JavaScript value to its equivalent DynamoDB AttributeValue type. @@ -37,6 +38,8 @@ export const convertToAttr = (data: NativeAttributeValue, options?: marshallOpti return { BOOL: data.valueOf() }; } else if (typeof data === "number" || data?.constructor?.name === "Number") { return convertToNumberAttr(data); + } else if (data instanceof NumberValue) { + return data.toAttributeValue(); } else if (typeof data === "bigint") { return convertToBigIntAttr(data); } else if (typeof data === "string" || data?.constructor?.name === "String") { @@ -76,7 +79,12 @@ const convertToSetAttr = ( } const item = setToOperate.values().next().value; - if (typeof item === "number") { + + if (item instanceof NumberValue) { + return { + NS: Array.from(setToOperate).map((_) => _.toString()), + }; + } else if (typeof item === "number") { return { NS: Array.from(setToOperate) .map(convertToNumberAttr) diff --git a/packages/util-dynamodb/src/convertToNative.ts b/packages/util-dynamodb/src/convertToNative.ts index 89ab5777fe6a..3b140d5c8dc5 100644 --- a/packages/util-dynamodb/src/convertToNative.ts +++ b/packages/util-dynamodb/src/convertToNative.ts @@ -1,6 +1,7 @@ import { AttributeValue } from "@aws-sdk/client-dynamodb"; -import { NativeAttributeValue, NumberValue } from "./models"; +import type { NativeAttributeValue, NumberValue as INumberValue } from "./models"; +import { NumberValue } from "./NumberValue"; import { unmarshallOptions } from "./unmarshall"; /** @@ -43,12 +44,15 @@ export const convertToNative = (data: AttributeValue, options?: unmarshallOption const convertNumber = (numString: string, options?: unmarshallOptions): number | bigint | NumberValue => { if (options?.wrapNumbers) { - return { value: numString }; + return NumberValue.from(numString); } const num = Number(numString); const infinityValues = [Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]; - if ((num > Number.MAX_SAFE_INTEGER || num < Number.MIN_SAFE_INTEGER) && !infinityValues.includes(num)) { + const isLargeFiniteNumber = + (num > Number.MAX_SAFE_INTEGER || num < Number.MIN_SAFE_INTEGER) && !infinityValues.includes(num); + + if (isLargeFiniteNumber) { if (typeof BigInt === "function") { try { return BigInt(numString); diff --git a/packages/util-dynamodb/src/index.ts b/packages/util-dynamodb/src/index.ts index 0d6f5d7852bd..60877561d334 100644 --- a/packages/util-dynamodb/src/index.ts +++ b/packages/util-dynamodb/src/index.ts @@ -1,3 +1,4 @@ +export { NumberValue as NumberValueImpl } from "./NumberValue"; export * from "./convertToAttr"; export * from "./convertToNative"; export * from "./marshall"; diff --git a/packages/util-dynamodb/src/unmarshall.ts b/packages/util-dynamodb/src/unmarshall.ts index c705e93ea70f..d6256fdc81ff 100644 --- a/packages/util-dynamodb/src/unmarshall.ts +++ b/packages/util-dynamodb/src/unmarshall.ts @@ -12,6 +12,7 @@ export interface unmarshallOptions { * This allows for the safe round-trip transport of numbers of arbitrary size. */ wrapNumbers?: boolean; + /** * When true, skip wrapping the data in `{ M: data }` before converting. *