From 3ea56140295901683e06d62f7b9d7738a2061ef2 Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Tue, 27 Aug 2024 13:33:25 +0100 Subject: [PATCH 1/4] If unmarshal results in a float64, return the string representation of a big int instead Signed-off-by: Matthew Whitehead --- pkg/fftypes/jsonany.go | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/pkg/fftypes/jsonany.go b/pkg/fftypes/jsonany.go index 05d1e1b..85f3120 100644 --- a/pkg/fftypes/jsonany.go +++ b/pkg/fftypes/jsonany.go @@ -21,6 +21,7 @@ import ( "crypto/sha256" "database/sql/driver" "encoding/json" + "math/big" "github.com/hyperledger/firefly-common/pkg/i18n" "github.com/hyperledger/firefly-common/pkg/log" @@ -76,7 +77,38 @@ func (h *JSONAny) Unmarshal(ctx context.Context, v interface{}) error { if h == nil { return i18n.NewError(ctx, i18n.MsgNilOrNullObject) } - return json.Unmarshal([]byte(*h), v) + + err := json.Unmarshal([]byte(*h), v) + if err != nil { + return err + } + + // To support large numbers, check if Go unmarshalled the data to a float64 and then + // unmarshal it to a string instead + if vt, ok := v.(*interface{}); ok { + if _, ok := (*vt).(float64); ok { + // If the value has unmarshalled to a float64 we can't be sure the number + // didn't overflow 2^64-1 so we'll use parseFloat on the original value + // and return the string representation of the number. + i := new(big.Int) + f, _, err := big.ParseFloat(h.String(), 10, 256, big.ToNearestEven) + if err != nil { + return err + } + i, accuracy := f.Int(i) + if accuracy != big.Exact { + // If we weren't able to decode without losing precision, return an error + return i18n.NewError(ctx, i18n.MsgBigIntParseFailed) + } + + err = json.Unmarshal([]byte("\""+i.String()+"\""), v) + if err != nil { + return err + } + } + } + + return err } func (h *JSONAny) Hash() *Bytes32 { From 28e4bc6195906d09880d6a32a47285a3f7e7315a Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Tue, 27 Aug 2024 14:41:50 +0100 Subject: [PATCH 2/4] Use decoder.UseNumber() to avoid unmarshalling to floats Signed-off-by: Matthew Whitehead --- pkg/fftypes/jsonany.go | 36 ++++++------------------------------ pkg/fftypes/jsonany_test.go | 13 +++++++++++++ 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/pkg/fftypes/jsonany.go b/pkg/fftypes/jsonany.go index 85f3120..7550ef4 100644 --- a/pkg/fftypes/jsonany.go +++ b/pkg/fftypes/jsonany.go @@ -1,4 +1,4 @@ -// Copyright © 2023 Kaleido, Inc. +// Copyright © 2024 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -21,7 +21,7 @@ import ( "crypto/sha256" "database/sql/driver" "encoding/json" - "math/big" + "strings" "github.com/hyperledger/firefly-common/pkg/i18n" "github.com/hyperledger/firefly-common/pkg/log" @@ -78,37 +78,13 @@ func (h *JSONAny) Unmarshal(ctx context.Context, v interface{}) error { return i18n.NewError(ctx, i18n.MsgNilOrNullObject) } - err := json.Unmarshal([]byte(*h), v) - if err != nil { + d := json.NewDecoder(strings.NewReader(h.String())) + d.UseNumber() + if err := d.Decode(v); err != nil { return err } - // To support large numbers, check if Go unmarshalled the data to a float64 and then - // unmarshal it to a string instead - if vt, ok := v.(*interface{}); ok { - if _, ok := (*vt).(float64); ok { - // If the value has unmarshalled to a float64 we can't be sure the number - // didn't overflow 2^64-1 so we'll use parseFloat on the original value - // and return the string representation of the number. - i := new(big.Int) - f, _, err := big.ParseFloat(h.String(), 10, 256, big.ToNearestEven) - if err != nil { - return err - } - i, accuracy := f.Int(i) - if accuracy != big.Exact { - // If we weren't able to decode without losing precision, return an error - return i18n.NewError(ctx, i18n.MsgBigIntParseFailed) - } - - err = json.Unmarshal([]byte("\""+i.String()+"\""), v) - if err != nil { - return err - } - } - } - - return err + return nil } func (h *JSONAny) Hash() *Bytes32 { diff --git a/pkg/fftypes/jsonany_test.go b/pkg/fftypes/jsonany_test.go index 9ed37c3..6827579 100644 --- a/pkg/fftypes/jsonany_test.go +++ b/pkg/fftypes/jsonany_test.go @@ -181,6 +181,19 @@ func TestUnmarshal(t *testing.T) { assert.Equal(t, "value1", myObj.Key1) } +func TestUnmarshalHugeNumber(t *testing.T) { + + var h *JSONAny + var myObj struct { + Key1 interface{} `json:"key1"` + } + + h = JSONAnyPtr(`{"key1":123456789123456789123456789}`) + err := h.Unmarshal(context.Background(), &myObj) + assert.NoError(t, err) + assert.Equal(t, json.Number("123456789123456789123456789"), myObj.Key1) +} + func TestNilHash(t *testing.T) { assert.Nil(t, (*JSONAny)(nil).Hash()) } From edb07289f1564c22623642b1f759fcea74f382eb Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Tue, 27 Aug 2024 14:48:06 +0100 Subject: [PATCH 3/4] Add error test case Signed-off-by: Matthew Whitehead --- pkg/fftypes/jsonany_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/fftypes/jsonany_test.go b/pkg/fftypes/jsonany_test.go index 6827579..8aab48e 100644 --- a/pkg/fftypes/jsonany_test.go +++ b/pkg/fftypes/jsonany_test.go @@ -194,6 +194,18 @@ func TestUnmarshalHugeNumber(t *testing.T) { assert.Equal(t, json.Number("123456789123456789123456789"), myObj.Key1) } +func TestUnmarshalHugeNumberError(t *testing.T) { + + var h *JSONAny + var myObj struct { + Key1 interface{} `json:"key1"` + } + + h = JSONAnyPtr(`{"key1":1234567891invalidchars234569}`) + err := h.Unmarshal(context.Background(), &myObj) + assert.Error(t, err) +} + func TestNilHash(t *testing.T) { assert.Nil(t, (*JSONAny)(nil).Hash()) } From 3165432b9ab5d68899461ca0dc05ffce6e084a9b Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Wed, 28 Aug 2024 10:10:26 +0100 Subject: [PATCH 4/4] Explicitly test for attempts to unmarshal to float64 and fail Signed-off-by: Matthew Whitehead --- pkg/fftypes/jsonany.go | 4 ++++ pkg/fftypes/jsonany_test.go | 24 ++++++++++++++++++++++-- pkg/i18n/en_base_error_messages.go | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pkg/fftypes/jsonany.go b/pkg/fftypes/jsonany.go index 7550ef4..7e3bfda 100644 --- a/pkg/fftypes/jsonany.go +++ b/pkg/fftypes/jsonany.go @@ -78,6 +78,10 @@ func (h *JSONAny) Unmarshal(ctx context.Context, v interface{}) error { return i18n.NewError(ctx, i18n.MsgNilOrNullObject) } + if _, ok := v.(*float64); ok { + return i18n.NewError(ctx, i18n.MsgUnmarshalToFloat64NotSupported) + } + d := json.NewDecoder(strings.NewReader(h.String())) d.UseNumber() if err := d.Decode(v); err != nil { diff --git a/pkg/fftypes/jsonany_test.go b/pkg/fftypes/jsonany_test.go index 8aab48e..1eb44b8 100644 --- a/pkg/fftypes/jsonany_test.go +++ b/pkg/fftypes/jsonany_test.go @@ -183,15 +183,35 @@ func TestUnmarshal(t *testing.T) { func TestUnmarshalHugeNumber(t *testing.T) { + var myInt64Variable int64 + var myFloat64Variable float64 + ctx := context.Background() var h *JSONAny var myObj struct { Key1 interface{} `json:"key1"` + Key2 JSONAny `json:"key2"` + Key3 JSONAny `json:"key3"` } - h = JSONAnyPtr(`{"key1":123456789123456789123456789}`) - err := h.Unmarshal(context.Background(), &myObj) + h = JSONAnyPtr(`{"key1":123456789123456789123456789, "key2":123456789123456789123456789, "key3":1234}`) + err := h.Unmarshal(ctx, &myObj) assert.NoError(t, err) assert.Equal(t, json.Number("123456789123456789123456789"), myObj.Key1) + + assert.NoError(t, err) + assert.Equal(t, "123456789123456789123456789", myObj.Key2.String()) + + err = myObj.Key2.Unmarshal(ctx, &myInt64Variable) + assert.Error(t, err) + assert.Regexp(t, "cannot unmarshal number 123456789123456789123456789 into Go value of type int64", err) + + err = myObj.Key3.Unmarshal(ctx, &myInt64Variable) + assert.NoError(t, err) + assert.Equal(t, int64(1234), myInt64Variable) + + err = myObj.Key2.Unmarshal(ctx, &myFloat64Variable) + assert.Error(t, err) + assert.Regexp(t, "FF00249", err) } func TestUnmarshalHugeNumberError(t *testing.T) { diff --git a/pkg/i18n/en_base_error_messages.go b/pkg/i18n/en_base_error_messages.go index 6a5219f..f82e399 100644 --- a/pkg/i18n/en_base_error_messages.go +++ b/pkg/i18n/en_base_error_messages.go @@ -183,4 +183,5 @@ var ( MsgDBExecFailed = ffe("FF00245", "Database update failed") MsgDBErrorBuildingStatement = ffe("FF00247", "Error building statement: %s") MsgDBReadInsertTSFailed = ffe("FF00248", "Failed to read timestamp from database optimized upsert: %s") + MsgUnmarshalToFloat64NotSupported = ffe("FF00249", "Unmarshalling to a float64 is not supported due to possible precision loss. Consider unmarshalling to an interface, json.Number or fftypes.JSONAny instead") )