feat(lib-dynamodb): large number handling (#5427)

* feat(lib-dynamodb): large number handling

* feat(lib-dynamodb): large number handling docs and set test case

* feat(lib-dynamodb): set release tag

* feat(lib-dynamodb): remove unsafe conversion feature

* feat(lib-dynamodb): add 1e100 number test case

* feat(lib-dynamodb): large number handling, remove extra unmarshall option

lib/lib-dynamodb/README.md

@@ -20,18 +20,18 @@ Responses from DynamoDB are unmarshalled into plain JavaScript objects
 by the `DocumentClient`. The `DocumentClient` does not accept
 `AttributeValue`s in favor of native JavaScript types.
 
-|          JavaScript Type          | DynamoDB AttributeValue |
-| :-------------------------------: | ----------------------- |
-|              String               | S                       |
-|          Number / BigInt          | N                       |
-|              Boolean              | BOOL                    |
-|               null                | NULL                    |
-|               Array               | L                       |
-|              Object               | M                       |
-|   Set\<Uint8Array, Blob, ...\>    | BS                      |
-|       Set\<Number, BigInt\>       | NS                      |
-|           Set\<String\>           | SS                      |
-| Uint8Array, Buffer, File, Blob... | B                       |
+|          JavaScript Type           | DynamoDB AttributeValue |
+| :--------------------------------: | ----------------------- |
+|               String               | S                       |
+|   Number / BigInt / NumberValue    | N                       |
+|              Boolean               | BOOL                    |
+|                null                | NULL                    |
+|               Array                | L                       |
+|               Object               | M                       |
+|    Set\<Uint8Array, Blob, ...\>    | BS                      |
+| Set\<Number, BigInt, NumberValue\> | NS                      |
+|           Set\<String\>            | SS                      |
+| Uint8Array, Buffer, File, Blob...  | B                       |
 
 ### Example
 
@@ -98,20 +98,48 @@ const ddbDocClient = DynamoDBDocument.from(client); // client is DynamoDB client
 The configuration for marshalling and unmarshalling can be sent as an optional
 second parameter during creation of document client as follows:
 
-```js
-const marshallOptions = {
-  // Whether to automatically convert empty strings, blobs, and sets to `null`.
-  convertEmptyValues: false, // false, by default.
-  // Whether to remove undefined values while marshalling.
-  removeUndefinedValues: false, // false, by default.
-  // Whether to convert typeof object to map attribute.
-  convertClassInstanceToMap: false, // false, by default.
-};
-
-const unmarshallOptions = {
-  // Whether to return numbers as a string instead of converting them to native JavaScript numbers.
-  wrapNumbers: false, // false, by default.
-};
+```ts
+export interface marshallOptions {
+  /**
+   * Whether to automatically convert empty strings, blobs, and sets to `null`
+   */
+  convertEmptyValues?: boolean;
+  /**
+   * Whether to remove undefined values while marshalling.
+   */
+  removeUndefinedValues?: boolean;
+  /**
+   * Whether to convert typeof object to map attribute.
+   */
+  convertClassInstanceToMap?: boolean;
+  /**
+   * Whether to convert the top level container
+   * if it is a map or list.
+   *
+   * Default is true when using the DynamoDBDocumentClient,
+   * but false if directly using the marshall function (backwards compatibility).
+   */
+  convertTopLevelContainer?: boolean;
+}
+
+export interface unmarshallOptions {
+  /**
+   * Whether to return numbers as a string instead of converting them to native JavaScript numbers.
+   * This allows for the safe round-trip transport of numbers of arbitrary size.
+   */
+  wrapNumbers?: boolean;
+
+  /**
+   * When true, skip wrapping the data in `{ M: data }` before converting.
+   *
+   * Default is true when using the DynamoDBDocumentClient,
+   * but false if directly using the unmarshall function (backwards compatibility).
+   */
+  convertWithoutMapWrapper?: boolean;
+}
+
+const marshallOptions: marshallOptions = {};
+const unmarshallOptions: unmarshallOptions = {};
 
 const translateConfig = { marshallOptions, unmarshallOptions };
 
@@ -160,6 +188,57 @@ await ddbDocClient.put({
 });
 ```
 
+### Large Numbers and `NumberValue`.
+
+On the input or marshalling side, the class `NumberValue` can be used
+anywhere to represent a DynamoDB number value, even small numbers.
+
+```ts
+import { DynamoDB } from "@aws-sdk/client-dynamodb";
+import { NumberValue, DynamoDBDocument } from "@aws-sdk/lib-dynamodb";
+
+// Note, the client will not validate the acceptability of the number
+// in terms of size or format.
+// It is only here to preserve your precise representation.
+const client = DynamoDBDocument.from(new DynamoDB({}));
+
+await client.put({
+  Item: {
+    id: 1,
+    smallNumber: NumberValue.from("123"),
+    bigNumber: NumberValue.from("1000000000000000000000.000000000001"),
+    nSet: new Set([123, NumberValue.from("456"), 789]),
+  },
+});
+```
+
+On the output or unmarshalling side, the class `NumberValue` is used
+depending on your setting for the `unmarshallOptions` flag `wrapnumbers`,
+shown above.
+
+```ts
+import { DynamoDB } from "@aws-sdk/client-dynamodb";
+import { NumberValue, DynamoDBDocument } from "@aws-sdk/lib-dynamodb";
+
+const client = DynamoDBDocument.from(new DynamoDB({}));
+
+const response = await client.get({
+  Key: {
+    id: 1,
+  },
+});
+
+/**
+ * Numbers in the response may be a number, a BigInt, or a NumberValue depending
+ * on how you set `wrapNumbers`.
+ */
+const value = response.Item.bigNumber;
+```
+
+`NumberValue` does not provide a way to do mathematical operations on itself.
+To do mathematical operations, take the string value of `NumberValue` by calling
+`.toString()` and supply it to your chosen big number implementation.
+
 ### Client and Command middleware stacks
 
 As with other AWS SDK for JavaScript v3 clients, you can apply middleware functions

lib/lib-dynamodb/src/index.ts

@@ -4,3 +4,5 @@ export * from "./DynamoDBDocumentClient";
 // smithy-typescript generated code
 export * from "./commands";
 export * from "./pagination";
+
+export { NumberValueImpl as NumberValue } from "@aws-sdk/util-dynamodb";

lib/lib-dynamodb/src/test/lib-dynamodb.e2e.spec.ts

@@ -15,6 +15,7 @@ import {
   ExecuteStatementCommandOutput,
   ExecuteTransactionCommandOutput,
   GetCommandOutput,
+  NumberValue,
   PutCommandOutput,
   QueryCommandOutput,
   ScanCommandOutput,
@@ -32,6 +33,9 @@ describe(DynamoDBDocument.name, () => {
     marshallOptions: {
       convertTopLevelContainer: true,
     },
+    unmarshallOptions: {
+      wrapNumbers: true,
+    },
   });
 
   function throwIfError(e: unknown) {
@@ -76,30 +80,39 @@ describe(DynamoDBDocument.name, () => {
   const data = {
     null: null,
     string: "myString",
-    number: 1,
+    number: NumberValue.from(1),
+    bigInt: NumberValue.from(
+      "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+    ),
+    bigNumber: NumberValue.from("3210000000000000000.0000000000000123"),
     boolean: true,
     sSet: new Set(["my", "string", "set"]),
-    nSet: new Set([2, 3, 4]),
+    nSet: new Set([2, 3, 4].map(NumberValue.from)),
     list: [
       null,
       "myString",
-      1,
+      NumberValue.from(1),
       true,
       new Set(["my", "string", "set"]),
-      new Set([2, 3, 4]),
-      ["listInList", 1, null],
+      new Set([NumberValue.from(2), NumberValue.from(3), NumberValue.from(4)]),
+      new Set([
+        NumberValue.from("3210000000000000000.0000000000000123"),
+        NumberValue.from("3210000000000000001.0000000000000123"),
+        NumberValue.from("3210000000000000002.0000000000000123"),
+      ]),
+      ["listInList", NumberValue.from(1), null],
       {
         mapInList: "mapInList",
       },
     ],
     map: {
       null: null,
       string: "myString",
-      number: 1,
+      number: NumberValue.from(1),
       boolean: true,
       sSet: new Set(["my", "string", "set"]),
-      nSet: new Set([2, 3, 4]),
-      listInMap: ["listInMap", 1, null],
+      nSet: new Set([2, 3, 4].map(NumberValue.from)),
+      listInMap: ["listInMap", NumberValue.from(1), null],
       mapInMap: { mapInMap: "mapInMap" },
     },
   };
@@ -116,6 +129,9 @@ describe(DynamoDBDocument.name, () => {
         if (input instanceof Set) {
           return new Set([...input].map(updateTransform)) as T;
         }
+        if (input instanceof NumberValue) {
+          return NumberValue.from(input.toString()) as T;
+        }
         return Object.entries(input).reduce((acc, [k, v]) => {
           acc[updateTransform(k)] = updateTransform(v);
           return acc;
@@ -436,28 +452,37 @@ describe(DynamoDBDocument.name, () => {
       expect(updateTransform(data)).toEqual({
         "null-x": null,
         "string-x": "myString-x",
-        "number-x": 2,
+        "number-x": NumberValue.from(1),
+        "bigInt-x": NumberValue.from(
+          "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+        ),
+        "bigNumber-x": NumberValue.from("3210000000000000000.0000000000000123"),
         "boolean-x": false,
         "sSet-x": new Set(["my-x", "string-x", "set-x"]),
-        "nSet-x": new Set([3, 4, 5]),
+        "nSet-x": new Set([2, 3, 4].map(NumberValue.from)),
         "list-x": [
           null,
           "myString-x",
-          2,
+          NumberValue.from(1),
           false,
           new Set(["my-x", "string-x", "set-x"]),
-          new Set([3, 4, 5]),
-          ["listInList-x", 2, null],
+          new Set([2, 3, 4].map(NumberValue.from)),
+          new Set([
+            NumberValue.from("3210000000000000000.0000000000000123"),
+            NumberValue.from("3210000000000000001.0000000000000123"),
+            NumberValue.from("3210000000000000002.0000000000000123"),
+          ]),
+          ["listInList-x", NumberValue.from(1), null],
           { "mapInList-x": "mapInList-x" },
         ],
         "map-x": {
           "null-x": null,
           "string-x": "myString-x",
-          "number-x": 2,
+          "number-x": NumberValue.from(1),
           "boolean-x": false,
           "sSet-x": new Set(["my-x", "string-x", "set-x"]),
-          "nSet-x": new Set([3, 4, 5]),
-          "listInMap-x": ["listInMap-x", 2, null],
+          "nSet-x": new Set([2, 3, 4].map(NumberValue.from)),
+          "listInMap-x": ["listInMap-x", NumberValue.from(1), null],
           "mapInMap-x": { "mapInMap-x": "mapInMap-x" },
         },
       });

packages/util-dynamodb/src/NumberValue.spec.ts

@@ -0,0 +1,38 @@
+import { NumberValue } from "./NumberValue";
+
+const BIG_DECIMAL =
+  "123456789012345678901234567890123456789012345678901234567890.123456789012345678901234567890123456789012345678901234567890";
+const BIG_INT = "123456789012345678901234567890123456789012345678901234567890";
+
+describe(NumberValue.name, () => {
+  it("can be statically constructed from numbers", () => {
+    expect(NumberValue.from(123.123).toString()).toEqual("123.123");
+
+    expect(() => NumberValue.from(1.23e100)).toThrow();
+    expect(() => NumberValue.from(Infinity)).toThrow();
+    expect(() => NumberValue.from(-Infinity)).toThrow();
+    expect(() => NumberValue.from(NaN)).toThrow();
+  });
+
+  it("can be statically constructed from strings", () => {
+    expect(NumberValue.from(BIG_DECIMAL).toString()).toEqual(BIG_DECIMAL);
+  });
+
+  it("can be statically constructed from BigInts", () => {
+    expect(NumberValue.from(BigInt(BIG_INT)).toString()).toEqual(BIG_INT);
+  });
+
+  it("can convert to AttributeValue", () => {
+    expect(NumberValue.from(BIG_DECIMAL).toAttributeValue()).toEqual({
+      N: BIG_DECIMAL,
+    });
+  });
+
+  it("can convert to string", () => {
+    expect(NumberValue.from(BIG_DECIMAL).toString()).toEqual(BIG_DECIMAL);
+  });
+
+  it("can convert to BigInt", () => {
+    expect(NumberValue.from(BIG_INT).toBigInt()).toEqual(BigInt(BIG_INT));
+  });
+});