From 76f6f32c583e960381fea76c778030839c065ef7 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 11 Jun 2024 14:48:47 -0400 Subject: [PATCH 01/63] feat: indexer base types --- indexer/base/README.md | 5 + indexer/base/column.go | 149 +++++++++++++++++ indexer/base/entity.go | 40 +++++ indexer/base/go.mod | 6 + indexer/base/go.sum | 0 indexer/base/kind.go | 304 ++++++++++++++++++++++++++++++++++ indexer/base/listener.go | 114 +++++++++++++ indexer/base/module_schema.go | 8 + indexer/base/table.go | 20 +++ 9 files changed, 646 insertions(+) create mode 100644 indexer/base/README.md create mode 100644 indexer/base/column.go create mode 100644 indexer/base/entity.go create mode 100644 indexer/base/go.mod create mode 100644 indexer/base/go.sum create mode 100644 indexer/base/kind.go create mode 100644 indexer/base/listener.go create mode 100644 indexer/base/module_schema.go create mode 100644 indexer/base/table.go diff --git a/indexer/base/README.md b/indexer/base/README.md new file mode 100644 index 000000000000..0b96a27dc63c --- /dev/null +++ b/indexer/base/README.md @@ -0,0 +1,5 @@ +# Indexer Base + +The indexer base module is designed to provide a stable, zero-dependency base layer for the built-in indexer functionality. Packages that integrate with the indexer should feel free to depend on this package without fear of any external dependencies being pulled in. + +The basic types for specifying index sources, targets and decoders are provided here along with a basic engine that ties these together. A package wishing to be an indexing source could accept an instance of `Engine` directly to be compatible with indexing. A package wishing to be a decoder can use the `Entity` and `Table` types. A package defining an indexing target should implement the `Indexer` interface. \ No newline at end of file diff --git a/indexer/base/column.go b/indexer/base/column.go new file mode 100644 index 000000000000..14fa4e55b322 --- /dev/null +++ b/indexer/base/column.go @@ -0,0 +1,149 @@ +package indexerbase + +import "fmt" + +// Column represents a column in a table schema. +type Column struct { + // Name is the name of the column. + Name string + + // Kind is the basic type of the column. + Kind Kind + + // Nullable indicates whether null values are accepted for the column. + Nullable bool + + // AddressPrefix is the address prefix of the column's kind, currently only used for Bech32AddressKind. + AddressPrefix string + + // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. + EnumDefinition EnumDefinition +} + +// EnumDefinition represents the definition of an enum type. +type EnumDefinition struct { + // Name is the name of the enum type. + Name string + + // Values is a list of distinct values that are part of the enum type. + Values []string +} + +// Validate validates the column. +func (c Column) Validate() error { + // non-empty name + if c.Name == "" { + return fmt.Errorf("column name cannot be empty") + } + + // valid kind + if err := c.Kind.Validate(); err != nil { + return fmt.Errorf("invalid column type for %q: %w", c.Name, err) + } + + // address prefix only valid with Bech32AddressKind + if c.Kind == Bech32AddressKind && c.AddressPrefix == "" { + return fmt.Errorf("missing address prefix for column %q", c.Name) + } else if c.Kind != Bech32AddressKind && c.AddressPrefix != "" { + return fmt.Errorf("address prefix is only valid for column %q with type Bech32AddressKind", c.Name) + } + + // enum definition only valid with EnumKind + if c.Kind == EnumKind { + if err := c.EnumDefinition.Validate(); err != nil { + return fmt.Errorf("invalid enum definition for column %q: %w", c.Name, err) + } + } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { + return fmt.Errorf("enum definition is only valid for column %q with type EnumKind", c.Name) + } + + return nil +} + +// Validate validates the enum definition. +func (e EnumDefinition) Validate() error { + if e.Name == "" { + return fmt.Errorf("enum definition name cannot be empty") + } + if len(e.Values) == 0 { + return fmt.Errorf("enum definition values cannot be empty") + } + seen := make(map[string]bool, len(e.Values)) + for i, v := range e.Values { + if v == "" { + return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name) + } + if seen[v] { + return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name) + } + seen[v] = true + } + return nil +} + +// ValidateValue validates that the value conforms to the column's kind and nullability. +func (c Column) ValidateValue(value any) error { + if value == nil { + if !c.Nullable { + return fmt.Errorf("column %q cannot be null", c.Name) + } + return nil + } + return c.Kind.ValidateValue(value) +} + +// ValidateKey validates that the value conforms to the set of columns as a Key in an EntityUpdate. +// See EntityUpdate.Key for documentation on the requirements of such values. +func ValidateKey(cols []Column, value any) error { + if len(cols) == 0 { + return nil + } + + if len(cols) == 1 { + return cols[0].ValidateValue(value) + } + + values, ok := value.([]any) + if !ok { + return fmt.Errorf("expected slice of values for key columns, got %T", value) + } + + if len(cols) != len(values) { + return fmt.Errorf("expected %d key columns, got %d values", len(cols), len(value.([]any))) + } + for i, col := range cols { + if err := col.ValidateValue(values[i]); err != nil { + return fmt.Errorf("invalid value for key column %q: %w", col.Name, err) + } + } + return nil +} + +// ValidateValue validates that the value conforms to the set of columns as a Value in an EntityUpdate. +// See EntityUpdate.Value for documentation on the requirements of such values. +func ValidateValue(cols []Column, value any) error { + valueUpdates, ok := value.(ValueUpdates) + if ok { + colMap := map[string]Column{} + for _, col := range cols { + colMap[col.Name] = col + } + var errs []error + valueUpdates.Iterate(func(colName string, value any) bool { + col, ok := colMap[colName] + if !ok { + errs = append(errs, fmt.Errorf("unknown column %q in value updates", colName)) + } + if err := col.ValidateValue(value); err != nil { + errs = append(errs, fmt.Errorf("invalid value for column %q: %w", colName, err)) + } + return true + }) + if len(errs) > 0 { + return fmt.Errorf("validation errors: %v", errs) + } + return nil + } else { + return ValidateKey(cols, value) + } +} diff --git a/indexer/base/entity.go b/indexer/base/entity.go new file mode 100644 index 000000000000..95f016037fd8 --- /dev/null +++ b/indexer/base/entity.go @@ -0,0 +1,40 @@ +package indexerbase + +// EntityUpdate represents an update operation on an entity in the schema. +type EntityUpdate struct { + // TableName is the name of the table that the entity belongs to in the schema. + TableName string + + // Key returns the value of the primary key of the entity and must conform to these constraints with respect + // that the schema that is defined for the entity: + // - if key represents a single column, then the value must be valid for the first column in that + // column list. For instance, if there is one column in the key of type String, then the value must be of + // type string + // - if key represents multiple columns, then the value must be a slice of values where each value is valid + // for the corresponding column in the column list. For instance, if there are two columns in the key of + // type String, String, then the value must be a slice of two strings. + // If the key has no columns, meaning that this is a singleton entity, then this value is ignored and can be nil. + Key any + + // Value returns the non-primary key columns of the entity and can either conform to the same constraints + // as EntityUpdate.Key or it may be and instance of ValueUpdates. ValueUpdates can be used as a performance + // optimization to avoid copying the values of the entity into the update and/or to omit unchanged columns. + // If this is a delete operation, then this value is ignored and can be nil. + Value any + + // Delete is a flag that indicates whether this update is a delete operation. If true, then the Value field + // is ignored and can be nil. + Delete bool +} + +// ValueUpdates is an interface that represents the value columns of an entity update. Columns that +// were not updated may be excluded from the update. Consumers should be aware that implementations +// may not filter out columns that were unchanged. However, if a column is omitted from the update +// it should be considered unchanged. +type ValueUpdates interface { + + // Iterate iterates over the columns and values in the entity update. The function should return + // true to continue iteration or false to stop iteration. Each column value should conform + // to the requirements of that column's type in the schema. + Iterate(func(col string, value any) bool) +} diff --git a/indexer/base/go.mod b/indexer/base/go.mod new file mode 100644 index 000000000000..c369648761e8 --- /dev/null +++ b/indexer/base/go.mod @@ -0,0 +1,6 @@ +module cosmossdk.io/indexer/base + +// NOTE: this go.mod should have zero dependencies and remain on an older version of Go +// to be compatible with legacy codebases. + +go 1.19 diff --git a/indexer/base/go.sum b/indexer/base/go.sum new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/indexer/base/kind.go b/indexer/base/kind.go new file mode 100644 index 000000000000..d1873944fb1b --- /dev/null +++ b/indexer/base/kind.go @@ -0,0 +1,304 @@ +package indexerbase + +import ( + "encoding/json" + "fmt" + "time" +) + +// Kind represents the basic type of a column in the table schema. +// Each kind defines the types of go values which should be accepted +// by listeners and generated by decoders when providing entity updates. +type Kind int + +const ( + // InvalidKind indicates that an invalid type. + InvalidKind Kind = iota + + // StringKind is a string type and values of this type must be of the go type string + // or implement fmt.Stringer(). + StringKind + + // BytesKind is a bytes type and values of this type must be of the go type []byte. + BytesKind + + // Int8Kind is an int8 type and values of this type must be of the go type int8. + Int8Kind + + // Uint8Kind is a uint8 type and values of this type must be of the go type uint8. + Uint8Kind + + // Int16Kind is an int16 type and values of this type must be of the go type int16. + Int16Kind + + // Uint16Kind is a uint16 type and values of this type must be of the go type uint16. + Uint16Kind + + // Int32Kind is an int32 type and values of this type must be of the go type int32. + Int32Kind + + // Uint32Kind is a uint32 type and values of this type must be of the go type uint32. + Uint32Kind + + // Int64Kind is an int64 type and values of this type must be of the go type int64. + Int64Kind + + // Uint64Kind is a uint64 type and values of this type must be of the go type uint64. + Uint64Kind + + // IntegerKind represents an arbitrary precision integer number. Values of this type must + // be of the go type string or a type that implements fmt.Stringer with the resulted string + // formatted as an integer number. + IntegerKind + + // DecimalKind represents an arbitrary precision decimal or integer number. Values of this type + // must be of the go type string or a type that implements fmt.Stringer with the resulting string + // formatted as decimal numbers with an optional fractional part. Exponential E-notation + // is supported but NaN and Infinity are not. + DecimalKind + + // BoolKind is a boolean type and values of this type must be of the go type bool. + BoolKind + + // TimeKind is a time type and values of this type must be of the go type time.Time. + TimeKind + + // DurationKind is a duration type and values of this type must be of the go type time.Duration. + DurationKind + + // Float32Kind is a float32 type and values of this type must be of the go type float32. + Float32Kind + + // Float64Kind is a float64 type and values of this type must be of the go type float64. + Float64Kind + + // Bech32AddressKind is a bech32 address type and values of this type must be of the go type string or []byte + // or a type which implements fmt.Stringer. Columns of this type are expected to set the AddressPrefix field + // in the column definition to the bech32 address prefix. + Bech32AddressKind + + // EnumKind is an enum type and values of this type must be of the go type string or implement fmt.Stringer. + // Columns of this type are expected to set the EnumDefinition field in the column definition to the enum + // definition. + EnumKind + + // JSONKind is a JSON type and values of this type can either be of go type json.RawMessage + // or any type that can be marshaled to JSON using json.Marshal. + JSONKind +) + +// Validate returns an error if the kind is invalid. +func (t Kind) Validate() error { + if t <= InvalidKind { + return fmt.Errorf("unknown type: %d", t) + } + if t > JSONKind { + return fmt.Errorf("invalid type: %d", t) + } + return nil +} + +// ValidateValue returns an error if the value does not the type go type specified by the kind. +// Some columns may accept nil values, however, this method does not have any notion of +// nullability. It only checks that the value is of the correct type. +func (t Kind) ValidateValue(value any) error { + switch t { + case StringKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case BytesKind: + _, ok := value.([]byte) + if !ok { + return fmt.Errorf("expected []byte, got %T", value) + } + case Int8Kind: + _, ok := value.(int8) + if !ok { + return fmt.Errorf("expected int8, got %T", value) + } + case Uint8Kind: + _, ok := value.(uint8) + if !ok { + return fmt.Errorf("expected uint8, got %T", value) + } + case Int16Kind: + _, ok := value.(int16) + if !ok { + return fmt.Errorf("expected int16, got %T", value) + } + case Uint16Kind: + _, ok := value.(uint16) + if !ok { + return fmt.Errorf("expected uint16, got %T", value) + } + case Int32Kind: + _, ok := value.(int32) + if !ok { + return fmt.Errorf("expected int32, got %T", value) + } + case Uint32Kind: + _, ok := value.(uint32) + if !ok { + return fmt.Errorf("expected uint32, got %T", value) + } + case Int64Kind: + _, ok := value.(int64) + if !ok { + return fmt.Errorf("expected int64, got %T", value) + } + case Uint64Kind: + _, ok := value.(uint64) + if !ok { + return fmt.Errorf("expected uint64, got %T", value) + } + case IntegerKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case DecimalKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case BoolKind: + _, ok := value.(bool) + if !ok { + return fmt.Errorf("expected bool, got %T", value) + } + case TimeKind: + _, ok := value.(time.Time) + if !ok { + return fmt.Errorf("expected time.Time, got %T", value) + } + case DurationKind: + _, ok := value.(time.Duration) + if !ok { + return fmt.Errorf("expected time.Duration, got %T", value) + } + case Float32Kind: + _, ok := value.(float32) + if !ok { + return fmt.Errorf("expected float32, got %T", value) + } + case Float64Kind: + _, ok := value.(float64) + if !ok { + return fmt.Errorf("expected float64, got %T", value) + } + case Bech32AddressKind: + _, ok := value.(string) + _, ok2 := value.([]byte) + _, ok3 := value.(fmt.Stringer) + if !ok && !ok2 && !ok3 { + return fmt.Errorf("expected string or []byte, got %T", value) + } + case EnumKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case JSONKind: + return nil + default: + return fmt.Errorf("invalid type: %d", t) + } + return nil +} + +// String returns a string representation of the kind. +func (t Kind) String() string { + switch t { + case StringKind: + return "string" + case BytesKind: + return "bytes" + case Int8Kind: + return "int8" + case Uint8Kind: + return "uint8" + case Int16Kind: + return "int16" + case Uint16Kind: + return "uint16" + case Int32Kind: + return "int32" + case Uint32Kind: + return "uint32" + case Int64Kind: + return "int64" + case Uint64Kind: + return "uint64" + case DecimalKind: + return "decimal" + case IntegerKind: + return "integer" + case BoolKind: + return "bool" + case TimeKind: + return "time" + case DurationKind: + return "duration" + case Float32Kind: + return "float32" + case Float64Kind: + return "float64" + case Bech32AddressKind: + return "bech32address" + case EnumKind: + return "enum" + case JSONKind: + return "json" + default: + return "" + } +} + +// KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, +// return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be +// represented as strings. It will return InvalidKind if the value is not a simple type. +func KindForGoValue(value any) Kind { + switch value.(type) { + case string, fmt.Stringer: + return StringKind + case []byte: + return BytesKind + case int8: + return Int8Kind + case uint8: + return Uint8Kind + case int16: + return Int16Kind + case uint16: + return Uint16Kind + case int32: + return Int32Kind + case uint32: + return Uint32Kind + case int64: + return Int64Kind + case uint64: + return Uint64Kind + case float32: + return Float32Kind + case float64: + return Float64Kind + case bool: + return BoolKind + case time.Time: + return TimeKind + case time.Duration: + return DurationKind + case json.RawMessage: + return JSONKind + default: + } + + return InvalidKind +} diff --git a/indexer/base/listener.go b/indexer/base/listener.go new file mode 100644 index 000000000000..cccd6f08e153 --- /dev/null +++ b/indexer/base/listener.go @@ -0,0 +1,114 @@ +package indexerbase + +import ( + "encoding/json" +) + +// Listener is an interface that defines methods for listening to both raw and logical blockchain data. +// It is valid for any of the methods to be nil, in which case the listener will not be called for that event. +// Listeners should understand the guarantees that are provided by the source they are listening to and +// understand which methods will or will not be called. For instance, most blockchains will not do logical +// decoding of data out of the box, so the EnsureLogicalSetup and OnEntityUpdate methods will not be called. +// These methods will only be called when listening logical decoding is setup. +type Listener struct { + // StartBlock is called at the beginning of processing a block. + StartBlock func(uint64) error + + // OnBlockHeader is called when a block header is received. + OnBlockHeader func(BlockHeaderData) error + + // OnTx is called when a transaction is received. + OnTx func(TxData) error + + // OnEvent is called when an event is received. + OnEvent func(EventData) error + + // OnKVPair is called when a key-value has been written to the store for a given module. + OnKVPair func(module string, key, value []byte, delete bool) error + + // Commit is called when state is commited, usually at the end of a block. Any + // indexers should commit their data when this is called and return an error if + // they are unable to commit. + Commit func() error + + // EnsureLogicalSetup should be called whenever the blockchain process starts OR whenever + // logical decoding of a module is initiated. An indexer listening to this event + // should ensure that they have performed whatever initialization steps (such as database + // migrations) required to receive OnEntityUpdate events for the given module. If the + // schema is incompatible with the existing schema, the listener should return an error. + // If the listener is persisting state for the module, it should return the last block + // that was saved for the module so that the framework can determine whether it is safe + // to resume indexing from the current height or whether there is a gap (usually an error). + // If the listener does not persist any state for the module, it should return 0 for lastBlock + // and nil for error. + // If the listener has initialized properly and would like to persist state for the module, + // but does not have any persisted state yet, it should return -1 for lastBlock and nil for error. + // In this case, the framework will perform a "catch-up sync" calling OnEntityUpdate for every + // entity already in the module followed by CommitCatchupSync before processing new block data. + EnsureLogicalSetup func(module string, schema ModuleSchema) (lastBlock int64, err error) + + // OnEntityUpdate is called whenever an entity is updated in the module. This is only called + // when logical data is available. It should be assumed that the same data in raw form + // is also passed to OnKVPair. + OnEntityUpdate func(module string, update EntityUpdate) error + + // CommitCatchupSync is called after all existing entities for a module have been passed to + // OnEntityUpdate during a catch-up sync which has been initiated by return -1 for lastBlock + // in EnsureLogicalSetup. The listener should commit all the data that has been received at + // this point and also save the block number as the last block that has been processed so + // that processing of regular block data can resume from this point in the future. + CommitCatchupSync func(module string, block uint64) error +} + +// BlockHeaderData represents the raw block header data that is passed to a listener. +type BlockHeaderData struct { + // Height is the height of the block. + Height uint64 + + // Bytes is the raw byte representation of the block header. + Bytes ToBytes + + // JSON is the JSON representation of the block header. It should generally be a JSON object. + JSON ToJSON +} + +// TxData represents the raw transaction data that is passed to a listener. +type TxData struct { + // TxIndex is the index of the transaction in the block. + TxIndex int32 + + // Bytes is the raw byte representation of the transaction. + Bytes ToBytes + + // JSON is the JSON representation of the transaction. It should generally be a JSON object. + JSON ToJSON +} + +// EventData represents event data that is passed to a listener. +type EventData struct { + // TxIndex is the index of the transaction in the block to which this event is associated. + // It should be set to a negative number if the event is not associated with a transaction. + // Canonically -1 should be used to represent begin block processing and -2 should be used to + // represent end block processing. + TxIndex int32 + + // MsgIndex is the index of the message in the transaction to which this event is associated. + // If TxIndex is negative, this index could correspond to the index of the message in + // begin or end block processing if such indexes exist, or it can be set to zero. + MsgIndex uint32 + + // EventIndex is the index of the event in the message to which this event is associated. + EventIndex uint32 + + // Type is the type of the event. + Type string + + // Data is the JSON representation of the event data. It should generally be a JSON object. + Data ToJSON +} + +// ToBytes is a function that lazily returns the raw byte representation of data. +type ToBytes = func() ([]byte, error) + +// ToJSON is a function that lazily returns the JSON representation of data. +type ToJSON = func() (json.RawMessage, error) diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go new file mode 100644 index 000000000000..4e8b81c2be3c --- /dev/null +++ b/indexer/base/module_schema.go @@ -0,0 +1,8 @@ +package indexerbase + +// ModuleSchema represents the logical schema of a module for purposes of indexing and querying. +type ModuleSchema struct { + + // Tables is a list of tables that are part of the schema for the module. + Tables []Table +} diff --git a/indexer/base/table.go b/indexer/base/table.go new file mode 100644 index 000000000000..2d076f6d3eb8 --- /dev/null +++ b/indexer/base/table.go @@ -0,0 +1,20 @@ +package indexerbase + +// Table represents a table in the schema of a module. +type Table struct { + // Name is the name of the table. + Name string + + // KeyColumns is a list of columns that make up the primary key of the table. + KeyColumns []Column + + // ValueColumns is a list of columns that are not part of the primary key of the table. + ValueColumns []Column + + // RetainDeletions is a flag that indicates whether the indexer should retain + // deleted rows in the database and flag them as deleted rather than actually + // deleting the row. For many types of data in state, the data is deleted even + // though it is still valid in order to save space. Indexers will want to have + // the option of retaining such data and distinguishing from other "true" deletions. + RetainDeletions bool +} From 63aeb85ecec56fd4cda2f21852ab5e9fde0b6770 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 11 Jun 2024 15:11:07 -0400 Subject: [PATCH 02/63] WIP on tests --- indexer/base/column.go | 4 +- indexer/base/column_test.go | 7 + indexer/base/kind.go | 19 ++- indexer/base/kind_test.go | 290 ++++++++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+), 8 deletions(-) create mode 100644 indexer/base/column_test.go create mode 100644 indexer/base/kind_test.go diff --git a/indexer/base/column.go b/indexer/base/column.go index 14fa4e55b322..30b75d161c82 100644 --- a/indexer/base/column.go +++ b/indexer/base/column.go @@ -82,6 +82,8 @@ func (e EnumDefinition) Validate() error { } // ValidateValue validates that the value conforms to the column's kind and nullability. +// It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind +// values are valid for their respective types behind conforming to the correct go type. func (c Column) ValidateValue(value any) error { if value == nil { if !c.Nullable { @@ -89,7 +91,7 @@ func (c Column) ValidateValue(value any) error { } return nil } - return c.Kind.ValidateValue(value) + return c.Kind.ValidateValueType(value) } // ValidateKey validates that the value conforms to the set of columns as a Key in an EntityUpdate. diff --git a/indexer/base/column_test.go b/indexer/base/column_test.go new file mode 100644 index 000000000000..b646247b058c --- /dev/null +++ b/indexer/base/column_test.go @@ -0,0 +1,7 @@ +package indexerbase + +import "testing" + +func TestColumnValidate(t *testing.T) { + +} diff --git a/indexer/base/kind.go b/indexer/base/kind.go index d1873944fb1b..0c86ab90f4c1 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -47,7 +47,7 @@ const ( Uint64Kind // IntegerKind represents an arbitrary precision integer number. Values of this type must - // be of the go type string or a type that implements fmt.Stringer with the resulted string + // be of the go type int64, string or a type that implements fmt.Stringer with the resulted string // formatted as an integer number. IntegerKind @@ -98,10 +98,12 @@ func (t Kind) Validate() error { return nil } -// ValidateValue returns an error if the value does not the type go type specified by the kind. +// ValidateValueType returns an error if the value does not the type go type specified by the kind. // Some columns may accept nil values, however, this method does not have any notion of // nullability. It only checks that the value is of the correct type. -func (t Kind) ValidateValue(value any) error { +// It also doesn't perform any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind +// values are valid for their respective types. +func (t Kind) ValidateValueType(value any) error { switch t { case StringKind: _, ok := value.(string) @@ -157,7 +159,8 @@ func (t Kind) ValidateValue(value any) error { case IntegerKind: _, ok := value.(string) _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { + _, ok3 := value.(int64) + if !ok && !ok2 && !ok3 { return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) } case DecimalKind: @@ -262,7 +265,10 @@ func (t Kind) String() string { // KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, // return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be -// represented as strings. It will return InvalidKind if the value is not a simple type. +// represented as strings. Generally all values which do not have a more specific type will +// return JSONKind because the framework cannot decide at this point whether the value +// can or cannot be marshaled to JSON. This method should generally only be used as a fallback +// when the kind of a column is not specified more specifically. func KindForGoValue(value any) Kind { switch value.(type) { case string, fmt.Stringer: @@ -298,7 +304,6 @@ func KindForGoValue(value any) Kind { case json.RawMessage: return JSONKind default: + return JSONKind } - - return InvalidKind } diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go new file mode 100644 index 000000000000..18075535b067 --- /dev/null +++ b/indexer/base/kind_test.go @@ -0,0 +1,290 @@ +package indexerbase + +import ( + "strings" + "testing" + "time" +) + +func TestKind_Validate(t *testing.T) { + validKinds := []Kind{ + StringKind, + BytesKind, + Int8Kind, + Uint8Kind, + Int16Kind, + Uint16Kind, + Int32Kind, + Uint32Kind, + Int64Kind, + Uint64Kind, + IntegerKind, + DecimalKind, + BoolKind, + EnumKind, + Bech32AddressKind, + } + + for _, kind := range validKinds { + if err := kind.Validate(); err != nil { + t.Errorf("expected valid kind %s to pass validation, got: %v", kind, err) + } + } + + invalidKinds := []Kind{ + Kind(-1), + InvalidKind, + Kind(100), + } + + for _, kind := range invalidKinds { + if err := kind.Validate(); err == nil { + t.Errorf("expected invalid kind %s to fail validation, got: %v", kind, err) + } + } +} + +func TestKind_ValidateValue(t *testing.T) { + tests := []struct { + kind Kind + value any + valid bool + }{ + { + kind: StringKind, + value: "hello", + valid: true, + }, + { + kind: StringKind, + value: &strings.Builder{}, + valid: true, + }, + { + kind: StringKind, + value: []byte("hello"), + valid: false, + }, + { + kind: BytesKind, + value: []byte("hello"), + valid: true, + }, + { + kind: BytesKind, + value: "hello", + valid: false, + }, + { + kind: Int8Kind, + value: int8(1), + valid: true, + }, + { + kind: Int8Kind, + value: int16(1), + valid: false, + }, + { + kind: Uint8Kind, + value: uint8(1), + valid: true, + }, + { + kind: Uint8Kind, + value: uint16(1), + valid: false, + }, + { + kind: Int16Kind, + value: int16(1), + valid: true, + }, + { + kind: Int16Kind, + value: int32(1), + valid: false, + }, + { + kind: Uint16Kind, + value: uint16(1), + valid: true, + }, + { + kind: Uint16Kind, + value: uint32(1), + valid: false, + }, + { + kind: Int32Kind, + value: int32(1), + valid: true, + }, + { + kind: Int32Kind, + value: int64(1), + valid: false, + }, + { + kind: Uint32Kind, + value: uint32(1), + valid: true, + }, + { + kind: Uint32Kind, + value: uint64(1), + valid: false, + }, + { + kind: Int64Kind, + value: int64(1), + valid: true, + }, + { + kind: Int64Kind, + value: int32(1), + valid: false, + }, + { + kind: Uint64Kind, + value: uint64(1), + valid: true, + }, + { + kind: Uint64Kind, + value: uint32(1), + valid: false, + }, + { + kind: IntegerKind, + value: "1", + valid: true, + }, + //{ + // kind: IntegerKind, + // value: (&strings.Builder{}).WriteString("1"), + // valid: true, + //}, + { + kind: IntegerKind, + value: int32(1), + valid: false, + }, + { + kind: IntegerKind, + value: int64(1), + valid: true, + }, + { + kind: DecimalKind, + value: "1.0", + valid: true, + }, + { + kind: DecimalKind, + value: "1", + valid: true, + }, + { + kind: DecimalKind, + value: "1.1e4", + valid: true, + }, + //{ + // kind: DecimalKind, + // value: (&strings.Builder{}).WriteString("1.0"), + // valid: true, + //}, + { + kind: DecimalKind, + value: int32(1), + valid: false, + }, + { + kind: Bech32AddressKind, + value: "cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h", + valid: true, + }, + //{ + // kind: Bech32AddressKind, + // value: (&strings.Builder{}).WriteString("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), + // valid: true, + //}, + { + kind: Bech32AddressKind, + value: 1, + valid: false, + }, + { + kind: BoolKind, + value: true, + valid: true, + }, + { + kind: BoolKind, + value: false, + valid: true, + }, + { + kind: BoolKind, + value: 1, + valid: false, + }, + { + kind: EnumKind, + value: "hello", + valid: true, + }, + //{ + // kind: EnumKind, + // value: (&strings.Builder{}).WriteString("hello"), + // valid: true, + //}, + { + kind: EnumKind, + value: 1, + valid: false, + }, + { + kind: TimeKind, + value: time.Now(), + valid: true, + }, + { + kind: TimeKind, + value: "hello", + valid: false, + }, + { + kind: DurationKind, + value: time.Second, + valid: true, + }, + { + kind: DurationKind, + value: "hello", + valid: false, + }, + { + kind: Float32Kind, + value: float32(1.0), + valid: true, + }, + { + kind: Float32Kind, + value: float64(1.0), + valid: false, + }, + // TODO float64, json + } + + for i, tt := range tests { + err := tt.kind.ValidateValueType(tt.value) + if tt.valid && err != nil { + t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) + } + if !tt.valid && err == nil { + t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) + } + } +} From 216e8f87298784fe65a0840ffa6a8743dad4cb31 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 12 Jun 2024 12:20:15 -0400 Subject: [PATCH 03/63] update listener --- indexer/base/listener.go | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/indexer/base/listener.go b/indexer/base/listener.go index cccd6f08e153..05b8de2a5013 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -8,9 +8,15 @@ import ( // It is valid for any of the methods to be nil, in which case the listener will not be called for that event. // Listeners should understand the guarantees that are provided by the source they are listening to and // understand which methods will or will not be called. For instance, most blockchains will not do logical -// decoding of data out of the box, so the EnsureLogicalSetup and OnEntityUpdate methods will not be called. +// decoding of data out of the box, so the InitializeModuleSchema and OnEntityUpdate methods will not be called. // These methods will only be called when listening logical decoding is setup. type Listener struct { + // Initialize is called when the listener is initialized before any other methods are called. + // The lastBlock return value should be the last block height the listener persisted if it is + // persisting block data, 0 if it is not interested in persisting block data, or -1 if it is + // persisting block data but has not persisted any data yet. + Initialize func(InitializationData) (lastBlock int64, err error) + // StartBlock is called at the beginning of processing a block. StartBlock func(uint64) error @@ -24,18 +30,19 @@ type Listener struct { OnEvent func(EventData) error // OnKVPair is called when a key-value has been written to the store for a given module. - OnKVPair func(module string, key, value []byte, delete bool) error + OnKVPair func(moduleName string, key, value []byte, delete bool) error // Commit is called when state is commited, usually at the end of a block. Any // indexers should commit their data when this is called and return an error if // they are unable to commit. Commit func() error - // EnsureLogicalSetup should be called whenever the blockchain process starts OR whenever + // InitializeModuleSchema should be called whenever the blockchain process starts OR whenever // logical decoding of a module is initiated. An indexer listening to this event // should ensure that they have performed whatever initialization steps (such as database // migrations) required to receive OnEntityUpdate events for the given module. If the - // schema is incompatible with the existing schema, the listener should return an error. + // indexer's schema is incompatible with the module's on-chain schema, the listener should return + // an error. // If the listener is persisting state for the module, it should return the last block // that was saved for the module so that the framework can determine whether it is safe // to resume indexing from the current height or whether there is a gap (usually an error). @@ -45,7 +52,7 @@ type Listener struct { // but does not have any persisted state yet, it should return -1 for lastBlock and nil for error. // In this case, the framework will perform a "catch-up sync" calling OnEntityUpdate for every // entity already in the module followed by CommitCatchupSync before processing new block data. - EnsureLogicalSetup func(module string, schema ModuleSchema) (lastBlock int64, err error) + InitializeModuleSchema func(module string, schema ModuleSchema) (lastBlock int64, err error) // OnEntityUpdate is called whenever an entity is updated in the module. This is only called // when logical data is available. It should be assumed that the same data in raw form @@ -54,10 +61,29 @@ type Listener struct { // CommitCatchupSync is called after all existing entities for a module have been passed to // OnEntityUpdate during a catch-up sync which has been initiated by return -1 for lastBlock - // in EnsureLogicalSetup. The listener should commit all the data that has been received at + // in InitializeModuleSchema. The listener should commit all the data that has been received at // this point and also save the block number as the last block that has been processed so // that processing of regular block data can resume from this point in the future. CommitCatchupSync func(module string, block uint64) error + + // SubscribedModules is a map of modules that the listener is interested in receiving events for in OnKVPair and + // logical decoding listeners (if these are registered). If this is left nil but listeners are registered, + // it is assumed that the listener is interested in all modules. + SubscribedModules map[string]bool +} + +// InitializationData represents initialization data that is passed to a listener. +type InitializationData struct { + + // HasEventAlignedWrites indicates that the blockchain data source will emit KV-pair events + // in an order aligned with transaction, message and event callbacks. If this is true + // then indexers can assume that KV-pair data is associated with these specific transactions, messages + // and events. This may be useful for indexers which store a log of all operations (such as immutable + // or version controlled databases) so that the history log can include fine grain correlation between + // state updates and transactions, messages and events. If this value is false, then indexers should + // assume that KV-pair data occurs out of order with respect to transaction, message and event callbacks - + // the only safe assumption being that KV-pair data is associated with the block in which it was emitted. + HasEventAlignedWrites bool } // BlockHeaderData represents the raw block header data that is passed to a listener. From 663ed17e3deca572e532e019b674ccae47657a82 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 11:26:30 -0400 Subject: [PATCH 04/63] rename column to field --- indexer/base/{column.go => field.go} | 78 +++++++++---------- .../base/{column_test.go => field_test.go} | 2 +- indexer/base/kind.go | 12 +-- indexer/base/table.go | 8 +- 4 files changed, 50 insertions(+), 50 deletions(-) rename indexer/base/{column.go => field.go} (53%) rename indexer/base/{column_test.go => field_test.go} (51%) diff --git a/indexer/base/column.go b/indexer/base/field.go similarity index 53% rename from indexer/base/column.go rename to indexer/base/field.go index 30b75d161c82..3dd3239c049c 100644 --- a/indexer/base/column.go +++ b/indexer/base/field.go @@ -2,18 +2,18 @@ package indexerbase import "fmt" -// Column represents a column in a table schema. -type Column struct { - // Name is the name of the column. +// Field represents a field in a table schema. +type Field struct { + // Name is the name of the field. Name string - // Kind is the basic type of the column. + // Kind is the basic type of the field. Kind Kind - // Nullable indicates whether null values are accepted for the column. + // Nullable indicates whether null values are accepted for the field. Nullable bool - // AddressPrefix is the address prefix of the column's kind, currently only used for Bech32AddressKind. + // AddressPrefix is the address prefix of the field's kind, currently only used for Bech32AddressKind. AddressPrefix string // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. @@ -29,32 +29,32 @@ type EnumDefinition struct { Values []string } -// Validate validates the column. -func (c Column) Validate() error { +// Validate validates the field. +func (c Field) Validate() error { // non-empty name if c.Name == "" { - return fmt.Errorf("column name cannot be empty") + return fmt.Errorf("field name cannot be empty") } // valid kind if err := c.Kind.Validate(); err != nil { - return fmt.Errorf("invalid column type for %q: %w", c.Name, err) + return fmt.Errorf("invalid field type for %q: %w", c.Name, err) } // address prefix only valid with Bech32AddressKind if c.Kind == Bech32AddressKind && c.AddressPrefix == "" { - return fmt.Errorf("missing address prefix for column %q", c.Name) + return fmt.Errorf("missing address prefix for field %q", c.Name) } else if c.Kind != Bech32AddressKind && c.AddressPrefix != "" { - return fmt.Errorf("address prefix is only valid for column %q with type Bech32AddressKind", c.Name) + return fmt.Errorf("address prefix is only valid for field %q with type Bech32AddressKind", c.Name) } // enum definition only valid with EnumKind if c.Kind == EnumKind { if err := c.EnumDefinition.Validate(); err != nil { - return fmt.Errorf("invalid enum definition for column %q: %w", c.Name, err) + return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err) } } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { - return fmt.Errorf("enum definition is only valid for column %q with type EnumKind", c.Name) + return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name) } return nil @@ -81,63 +81,63 @@ func (e EnumDefinition) Validate() error { return nil } -// ValidateValue validates that the value conforms to the column's kind and nullability. +// ValidateValue validates that the value conforms to the field's kind and nullability. // It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind // values are valid for their respective types behind conforming to the correct go type. -func (c Column) ValidateValue(value any) error { +func (c Field) ValidateValue(value any) error { if value == nil { if !c.Nullable { - return fmt.Errorf("column %q cannot be null", c.Name) + return fmt.Errorf("field %q cannot be null", c.Name) } return nil } return c.Kind.ValidateValueType(value) } -// ValidateKey validates that the value conforms to the set of columns as a Key in an EntityUpdate. +// ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. // See EntityUpdate.Key for documentation on the requirements of such values. -func ValidateKey(cols []Column, value any) error { - if len(cols) == 0 { +func ValidateKey(fields []Field, value any) error { + if len(fields) == 0 { return nil } - if len(cols) == 1 { - return cols[0].ValidateValue(value) + if len(fields) == 1 { + return fields[0].ValidateValue(value) } values, ok := value.([]any) if !ok { - return fmt.Errorf("expected slice of values for key columns, got %T", value) + return fmt.Errorf("expected slice of values for key fields, got %T", value) } - if len(cols) != len(values) { - return fmt.Errorf("expected %d key columns, got %d values", len(cols), len(value.([]any))) + if len(fields) != len(values) { + return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]any))) } - for i, col := range cols { - if err := col.ValidateValue(values[i]); err != nil { - return fmt.Errorf("invalid value for key column %q: %w", col.Name, err) + for i, field := range fields { + if err := field.ValidateValue(values[i]); err != nil { + return fmt.Errorf("invalid value for key field %q: %w", field.Name, err) } } return nil } -// ValidateValue validates that the value conforms to the set of columns as a Value in an EntityUpdate. +// ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate. // See EntityUpdate.Value for documentation on the requirements of such values. -func ValidateValue(cols []Column, value any) error { +func ValidateValue(fields []Field, value any) error { valueUpdates, ok := value.(ValueUpdates) if ok { - colMap := map[string]Column{} - for _, col := range cols { - colMap[col.Name] = col + fieldMap := map[string]Field{} + for _, field := range fields { + fieldMap[field.Name] = field } var errs []error - valueUpdates.Iterate(func(colName string, value any) bool { - col, ok := colMap[colName] + valueUpdates.Iterate(func(fieldName string, value any) bool { + field, ok := fieldMap[fieldName] if !ok { - errs = append(errs, fmt.Errorf("unknown column %q in value updates", colName)) + errs = append(errs, fmt.Errorf("unknown field %q in value updates", fieldName)) } - if err := col.ValidateValue(value); err != nil { - errs = append(errs, fmt.Errorf("invalid value for column %q: %w", colName, err)) + if err := field.ValidateValue(value); err != nil { + errs = append(errs, fmt.Errorf("invalid value for field %q: %w", fieldName, err)) } return true }) @@ -146,6 +146,6 @@ func ValidateValue(cols []Column, value any) error { } return nil } else { - return ValidateKey(cols, value) + return ValidateKey(fields, value) } } diff --git a/indexer/base/column_test.go b/indexer/base/field_test.go similarity index 51% rename from indexer/base/column_test.go rename to indexer/base/field_test.go index b646247b058c..09f4488bef38 100644 --- a/indexer/base/column_test.go +++ b/indexer/base/field_test.go @@ -2,6 +2,6 @@ package indexerbase import "testing" -func TestColumnValidate(t *testing.T) { +func TestField_Validate(t *testing.T) { } diff --git a/indexer/base/kind.go b/indexer/base/kind.go index 0c86ab90f4c1..bb888e3d8a1f 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -6,7 +6,7 @@ import ( "time" ) -// Kind represents the basic type of a column in the table schema. +// Kind represents the basic type of a field in the table schema. // Each kind defines the types of go values which should be accepted // by listeners and generated by decoders when providing entity updates. type Kind int @@ -73,12 +73,12 @@ const ( Float64Kind // Bech32AddressKind is a bech32 address type and values of this type must be of the go type string or []byte - // or a type which implements fmt.Stringer. Columns of this type are expected to set the AddressPrefix field - // in the column definition to the bech32 address prefix. + // or a type which implements fmt.Stringer. Fields of this type are expected to set the AddressPrefix field + // in the field definition to the bech32 address prefix. Bech32AddressKind // EnumKind is an enum type and values of this type must be of the go type string or implement fmt.Stringer. - // Columns of this type are expected to set the EnumDefinition field in the column definition to the enum + // Fields of this type are expected to set the EnumDefinition field in the field definition to the enum // definition. EnumKind @@ -99,7 +99,7 @@ func (t Kind) Validate() error { } // ValidateValueType returns an error if the value does not the type go type specified by the kind. -// Some columns may accept nil values, however, this method does not have any notion of +// Some fields may accept nil values, however, this method does not have any notion of // nullability. It only checks that the value is of the correct type. // It also doesn't perform any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind // values are valid for their respective types. @@ -268,7 +268,7 @@ func (t Kind) String() string { // represented as strings. Generally all values which do not have a more specific type will // return JSONKind because the framework cannot decide at this point whether the value // can or cannot be marshaled to JSON. This method should generally only be used as a fallback -// when the kind of a column is not specified more specifically. +// when the kind of a field is not specified more specifically. func KindForGoValue(value any) Kind { switch value.(type) { case string, fmt.Stringer: diff --git a/indexer/base/table.go b/indexer/base/table.go index 2d076f6d3eb8..350d961c2411 100644 --- a/indexer/base/table.go +++ b/indexer/base/table.go @@ -5,11 +5,11 @@ type Table struct { // Name is the name of the table. Name string - // KeyColumns is a list of columns that make up the primary key of the table. - KeyColumns []Column + // KeyFields is a list of fields that make up the primary key of the table. + KeyFields []Field - // ValueColumns is a list of columns that are not part of the primary key of the table. - ValueColumns []Column + // ValueFields is a list of fields that are not part of the primary key of the table. + ValueFields []Field // RetainDeletions is a flag that indicates whether the indexer should retain // deleted rows in the database and flag them as deleted rather than actually From 43113570d1b5da6b11088830c2447c08b271f99c Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 11:39:06 -0400 Subject: [PATCH 05/63] delete code, simplify --- indexer/base/field.go | 123 ---------------- indexer/base/field_test.go | 7 - indexer/base/kind.go | 227 ----------------------------- indexer/base/kind_test.go | 290 ------------------------------------- indexer/base/listener.go | 18 +-- 5 files changed, 5 insertions(+), 660 deletions(-) delete mode 100644 indexer/base/field_test.go delete mode 100644 indexer/base/kind_test.go diff --git a/indexer/base/field.go b/indexer/base/field.go index 3dd3239c049c..93755f47e6c4 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -1,7 +1,5 @@ package indexerbase -import "fmt" - // Field represents a field in a table schema. type Field struct { // Name is the name of the field. @@ -28,124 +26,3 @@ type EnumDefinition struct { // Values is a list of distinct values that are part of the enum type. Values []string } - -// Validate validates the field. -func (c Field) Validate() error { - // non-empty name - if c.Name == "" { - return fmt.Errorf("field name cannot be empty") - } - - // valid kind - if err := c.Kind.Validate(); err != nil { - return fmt.Errorf("invalid field type for %q: %w", c.Name, err) - } - - // address prefix only valid with Bech32AddressKind - if c.Kind == Bech32AddressKind && c.AddressPrefix == "" { - return fmt.Errorf("missing address prefix for field %q", c.Name) - } else if c.Kind != Bech32AddressKind && c.AddressPrefix != "" { - return fmt.Errorf("address prefix is only valid for field %q with type Bech32AddressKind", c.Name) - } - - // enum definition only valid with EnumKind - if c.Kind == EnumKind { - if err := c.EnumDefinition.Validate(); err != nil { - return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err) - } - } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { - return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name) - } - - return nil -} - -// Validate validates the enum definition. -func (e EnumDefinition) Validate() error { - if e.Name == "" { - return fmt.Errorf("enum definition name cannot be empty") - } - if len(e.Values) == 0 { - return fmt.Errorf("enum definition values cannot be empty") - } - seen := make(map[string]bool, len(e.Values)) - for i, v := range e.Values { - if v == "" { - return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name) - } - if seen[v] { - return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name) - } - seen[v] = true - } - return nil -} - -// ValidateValue validates that the value conforms to the field's kind and nullability. -// It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind -// values are valid for their respective types behind conforming to the correct go type. -func (c Field) ValidateValue(value any) error { - if value == nil { - if !c.Nullable { - return fmt.Errorf("field %q cannot be null", c.Name) - } - return nil - } - return c.Kind.ValidateValueType(value) -} - -// ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. -// See EntityUpdate.Key for documentation on the requirements of such values. -func ValidateKey(fields []Field, value any) error { - if len(fields) == 0 { - return nil - } - - if len(fields) == 1 { - return fields[0].ValidateValue(value) - } - - values, ok := value.([]any) - if !ok { - return fmt.Errorf("expected slice of values for key fields, got %T", value) - } - - if len(fields) != len(values) { - return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]any))) - } - for i, field := range fields { - if err := field.ValidateValue(values[i]); err != nil { - return fmt.Errorf("invalid value for key field %q: %w", field.Name, err) - } - } - return nil -} - -// ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate. -// See EntityUpdate.Value for documentation on the requirements of such values. -func ValidateValue(fields []Field, value any) error { - valueUpdates, ok := value.(ValueUpdates) - if ok { - fieldMap := map[string]Field{} - for _, field := range fields { - fieldMap[field.Name] = field - } - var errs []error - valueUpdates.Iterate(func(fieldName string, value any) bool { - field, ok := fieldMap[fieldName] - if !ok { - errs = append(errs, fmt.Errorf("unknown field %q in value updates", fieldName)) - } - if err := field.ValidateValue(value); err != nil { - errs = append(errs, fmt.Errorf("invalid value for field %q: %w", fieldName, err)) - } - return true - }) - if len(errs) > 0 { - return fmt.Errorf("validation errors: %v", errs) - } - return nil - } else { - return ValidateKey(fields, value) - } -} diff --git a/indexer/base/field_test.go b/indexer/base/field_test.go deleted file mode 100644 index 09f4488bef38..000000000000 --- a/indexer/base/field_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package indexerbase - -import "testing" - -func TestField_Validate(t *testing.T) { - -} diff --git a/indexer/base/kind.go b/indexer/base/kind.go index bb888e3d8a1f..ba4046ba330b 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -1,11 +1,5 @@ package indexerbase -import ( - "encoding/json" - "fmt" - "time" -) - // Kind represents the basic type of a field in the table schema. // Each kind defines the types of go values which should be accepted // by listeners and generated by decoders when providing entity updates. @@ -86,224 +80,3 @@ const ( // or any type that can be marshaled to JSON using json.Marshal. JSONKind ) - -// Validate returns an error if the kind is invalid. -func (t Kind) Validate() error { - if t <= InvalidKind { - return fmt.Errorf("unknown type: %d", t) - } - if t > JSONKind { - return fmt.Errorf("invalid type: %d", t) - } - return nil -} - -// ValidateValueType returns an error if the value does not the type go type specified by the kind. -// Some fields may accept nil values, however, this method does not have any notion of -// nullability. It only checks that the value is of the correct type. -// It also doesn't perform any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind -// values are valid for their respective types. -func (t Kind) ValidateValueType(value any) error { - switch t { - case StringKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) - } - case BytesKind: - _, ok := value.([]byte) - if !ok { - return fmt.Errorf("expected []byte, got %T", value) - } - case Int8Kind: - _, ok := value.(int8) - if !ok { - return fmt.Errorf("expected int8, got %T", value) - } - case Uint8Kind: - _, ok := value.(uint8) - if !ok { - return fmt.Errorf("expected uint8, got %T", value) - } - case Int16Kind: - _, ok := value.(int16) - if !ok { - return fmt.Errorf("expected int16, got %T", value) - } - case Uint16Kind: - _, ok := value.(uint16) - if !ok { - return fmt.Errorf("expected uint16, got %T", value) - } - case Int32Kind: - _, ok := value.(int32) - if !ok { - return fmt.Errorf("expected int32, got %T", value) - } - case Uint32Kind: - _, ok := value.(uint32) - if !ok { - return fmt.Errorf("expected uint32, got %T", value) - } - case Int64Kind: - _, ok := value.(int64) - if !ok { - return fmt.Errorf("expected int64, got %T", value) - } - case Uint64Kind: - _, ok := value.(uint64) - if !ok { - return fmt.Errorf("expected uint64, got %T", value) - } - case IntegerKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - _, ok3 := value.(int64) - if !ok && !ok2 && !ok3 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) - } - case DecimalKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) - } - case BoolKind: - _, ok := value.(bool) - if !ok { - return fmt.Errorf("expected bool, got %T", value) - } - case TimeKind: - _, ok := value.(time.Time) - if !ok { - return fmt.Errorf("expected time.Time, got %T", value) - } - case DurationKind: - _, ok := value.(time.Duration) - if !ok { - return fmt.Errorf("expected time.Duration, got %T", value) - } - case Float32Kind: - _, ok := value.(float32) - if !ok { - return fmt.Errorf("expected float32, got %T", value) - } - case Float64Kind: - _, ok := value.(float64) - if !ok { - return fmt.Errorf("expected float64, got %T", value) - } - case Bech32AddressKind: - _, ok := value.(string) - _, ok2 := value.([]byte) - _, ok3 := value.(fmt.Stringer) - if !ok && !ok2 && !ok3 { - return fmt.Errorf("expected string or []byte, got %T", value) - } - case EnumKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) - } - case JSONKind: - return nil - default: - return fmt.Errorf("invalid type: %d", t) - } - return nil -} - -// String returns a string representation of the kind. -func (t Kind) String() string { - switch t { - case StringKind: - return "string" - case BytesKind: - return "bytes" - case Int8Kind: - return "int8" - case Uint8Kind: - return "uint8" - case Int16Kind: - return "int16" - case Uint16Kind: - return "uint16" - case Int32Kind: - return "int32" - case Uint32Kind: - return "uint32" - case Int64Kind: - return "int64" - case Uint64Kind: - return "uint64" - case DecimalKind: - return "decimal" - case IntegerKind: - return "integer" - case BoolKind: - return "bool" - case TimeKind: - return "time" - case DurationKind: - return "duration" - case Float32Kind: - return "float32" - case Float64Kind: - return "float64" - case Bech32AddressKind: - return "bech32address" - case EnumKind: - return "enum" - case JSONKind: - return "json" - default: - return "" - } -} - -// KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, -// return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be -// represented as strings. Generally all values which do not have a more specific type will -// return JSONKind because the framework cannot decide at this point whether the value -// can or cannot be marshaled to JSON. This method should generally only be used as a fallback -// when the kind of a field is not specified more specifically. -func KindForGoValue(value any) Kind { - switch value.(type) { - case string, fmt.Stringer: - return StringKind - case []byte: - return BytesKind - case int8: - return Int8Kind - case uint8: - return Uint8Kind - case int16: - return Int16Kind - case uint16: - return Uint16Kind - case int32: - return Int32Kind - case uint32: - return Uint32Kind - case int64: - return Int64Kind - case uint64: - return Uint64Kind - case float32: - return Float32Kind - case float64: - return Float64Kind - case bool: - return BoolKind - case time.Time: - return TimeKind - case time.Duration: - return DurationKind - case json.RawMessage: - return JSONKind - default: - return JSONKind - } -} diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go deleted file mode 100644 index 18075535b067..000000000000 --- a/indexer/base/kind_test.go +++ /dev/null @@ -1,290 +0,0 @@ -package indexerbase - -import ( - "strings" - "testing" - "time" -) - -func TestKind_Validate(t *testing.T) { - validKinds := []Kind{ - StringKind, - BytesKind, - Int8Kind, - Uint8Kind, - Int16Kind, - Uint16Kind, - Int32Kind, - Uint32Kind, - Int64Kind, - Uint64Kind, - IntegerKind, - DecimalKind, - BoolKind, - EnumKind, - Bech32AddressKind, - } - - for _, kind := range validKinds { - if err := kind.Validate(); err != nil { - t.Errorf("expected valid kind %s to pass validation, got: %v", kind, err) - } - } - - invalidKinds := []Kind{ - Kind(-1), - InvalidKind, - Kind(100), - } - - for _, kind := range invalidKinds { - if err := kind.Validate(); err == nil { - t.Errorf("expected invalid kind %s to fail validation, got: %v", kind, err) - } - } -} - -func TestKind_ValidateValue(t *testing.T) { - tests := []struct { - kind Kind - value any - valid bool - }{ - { - kind: StringKind, - value: "hello", - valid: true, - }, - { - kind: StringKind, - value: &strings.Builder{}, - valid: true, - }, - { - kind: StringKind, - value: []byte("hello"), - valid: false, - }, - { - kind: BytesKind, - value: []byte("hello"), - valid: true, - }, - { - kind: BytesKind, - value: "hello", - valid: false, - }, - { - kind: Int8Kind, - value: int8(1), - valid: true, - }, - { - kind: Int8Kind, - value: int16(1), - valid: false, - }, - { - kind: Uint8Kind, - value: uint8(1), - valid: true, - }, - { - kind: Uint8Kind, - value: uint16(1), - valid: false, - }, - { - kind: Int16Kind, - value: int16(1), - valid: true, - }, - { - kind: Int16Kind, - value: int32(1), - valid: false, - }, - { - kind: Uint16Kind, - value: uint16(1), - valid: true, - }, - { - kind: Uint16Kind, - value: uint32(1), - valid: false, - }, - { - kind: Int32Kind, - value: int32(1), - valid: true, - }, - { - kind: Int32Kind, - value: int64(1), - valid: false, - }, - { - kind: Uint32Kind, - value: uint32(1), - valid: true, - }, - { - kind: Uint32Kind, - value: uint64(1), - valid: false, - }, - { - kind: Int64Kind, - value: int64(1), - valid: true, - }, - { - kind: Int64Kind, - value: int32(1), - valid: false, - }, - { - kind: Uint64Kind, - value: uint64(1), - valid: true, - }, - { - kind: Uint64Kind, - value: uint32(1), - valid: false, - }, - { - kind: IntegerKind, - value: "1", - valid: true, - }, - //{ - // kind: IntegerKind, - // value: (&strings.Builder{}).WriteString("1"), - // valid: true, - //}, - { - kind: IntegerKind, - value: int32(1), - valid: false, - }, - { - kind: IntegerKind, - value: int64(1), - valid: true, - }, - { - kind: DecimalKind, - value: "1.0", - valid: true, - }, - { - kind: DecimalKind, - value: "1", - valid: true, - }, - { - kind: DecimalKind, - value: "1.1e4", - valid: true, - }, - //{ - // kind: DecimalKind, - // value: (&strings.Builder{}).WriteString("1.0"), - // valid: true, - //}, - { - kind: DecimalKind, - value: int32(1), - valid: false, - }, - { - kind: Bech32AddressKind, - value: "cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h", - valid: true, - }, - //{ - // kind: Bech32AddressKind, - // value: (&strings.Builder{}).WriteString("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), - // valid: true, - //}, - { - kind: Bech32AddressKind, - value: 1, - valid: false, - }, - { - kind: BoolKind, - value: true, - valid: true, - }, - { - kind: BoolKind, - value: false, - valid: true, - }, - { - kind: BoolKind, - value: 1, - valid: false, - }, - { - kind: EnumKind, - value: "hello", - valid: true, - }, - //{ - // kind: EnumKind, - // value: (&strings.Builder{}).WriteString("hello"), - // valid: true, - //}, - { - kind: EnumKind, - value: 1, - valid: false, - }, - { - kind: TimeKind, - value: time.Now(), - valid: true, - }, - { - kind: TimeKind, - value: "hello", - valid: false, - }, - { - kind: DurationKind, - value: time.Second, - valid: true, - }, - { - kind: DurationKind, - value: "hello", - valid: false, - }, - { - kind: Float32Kind, - value: float32(1.0), - valid: true, - }, - { - kind: Float32Kind, - value: float64(1.0), - valid: false, - }, - // TODO float64, json - } - - for i, tt := range tests { - err := tt.kind.ValidateValueType(tt.value) - if tt.valid && err != nil { - t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) - } - if !tt.valid && err == nil { - t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) - } - } -} diff --git a/indexer/base/listener.go b/indexer/base/listener.go index 05b8de2a5013..493f1269096f 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -12,10 +12,11 @@ import ( // These methods will only be called when listening logical decoding is setup. type Listener struct { // Initialize is called when the listener is initialized before any other methods are called. - // The lastBlock return value should be the last block height the listener persisted if it is + // The lastBlockPersisted return value should be the last block height the listener persisted if it is // persisting block data, 0 if it is not interested in persisting block data, or -1 if it is - // persisting block data but has not persisted any data yet. - Initialize func(InitializationData) (lastBlock int64, err error) + // persisting block data but has not persisted any data yet. This check allows the indexer + // framework to ensure that the listener has not missed blocks. + Initialize func(InitializationData) (lastBlockPersisted int64, err error) // StartBlock is called at the beginning of processing a block. StartBlock func(uint64) error @@ -43,16 +44,7 @@ type Listener struct { // migrations) required to receive OnEntityUpdate events for the given module. If the // indexer's schema is incompatible with the module's on-chain schema, the listener should return // an error. - // If the listener is persisting state for the module, it should return the last block - // that was saved for the module so that the framework can determine whether it is safe - // to resume indexing from the current height or whether there is a gap (usually an error). - // If the listener does not persist any state for the module, it should return 0 for lastBlock - // and nil for error. - // If the listener has initialized properly and would like to persist state for the module, - // but does not have any persisted state yet, it should return -1 for lastBlock and nil for error. - // In this case, the framework will perform a "catch-up sync" calling OnEntityUpdate for every - // entity already in the module followed by CommitCatchupSync before processing new block data. - InitializeModuleSchema func(module string, schema ModuleSchema) (lastBlock int64, err error) + InitializeModuleSchema func(module string, schema ModuleSchema) error // OnEntityUpdate is called whenever an entity is updated in the module. This is only called // when logical data is available. It should be assumed that the same data in raw form From c52655a79c62f4b9bf4344b045e504f6e467c654 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 11:42:26 -0400 Subject: [PATCH 06/63] add error return --- indexer/base/entity.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/indexer/base/entity.go b/indexer/base/entity.go index 95f016037fd8..df2ee2d3dd2f 100644 --- a/indexer/base/entity.go +++ b/indexer/base/entity.go @@ -35,6 +35,7 @@ type ValueUpdates interface { // Iterate iterates over the columns and values in the entity update. The function should return // true to continue iteration or false to stop iteration. Each column value should conform - // to the requirements of that column's type in the schema. - Iterate(func(col string, value any) bool) + // to the requirements of that column's type in the schema. Iterate returns an error if + // it was unable to decode the values properly (which could be the case in lazy evaluation). + Iterate(func(col string, value any) bool) error } From 46669d35742ed17c39383d9fe92490faeb6f17bd Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 11:53:07 -0400 Subject: [PATCH 07/63] remove ability to filter subscribed modules - this is a bit dangerous --- indexer/base/listener.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/indexer/base/listener.go b/indexer/base/listener.go index 493f1269096f..43a66f450e76 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -57,11 +57,6 @@ type Listener struct { // this point and also save the block number as the last block that has been processed so // that processing of regular block data can resume from this point in the future. CommitCatchupSync func(module string, block uint64) error - - // SubscribedModules is a map of modules that the listener is interested in receiving events for in OnKVPair and - // logical decoding listeners (if these are registered). If this is left nil but listeners are registered, - // it is assumed that the listener is interested in all modules. - SubscribedModules map[string]bool } // InitializationData represents initialization data that is passed to a listener. From 0a47c393ac374be23d13deae32b8e2024de2361d Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 11:56:17 -0400 Subject: [PATCH 08/63] add docs about fields --- indexer/base/table.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/indexer/base/table.go b/indexer/base/table.go index 350d961c2411..f2061005cd9a 100644 --- a/indexer/base/table.go +++ b/indexer/base/table.go @@ -6,9 +6,12 @@ type Table struct { Name string // KeyFields is a list of fields that make up the primary key of the table. + // It can be empty in which case indexers should assume that this table is + // a singleton and ony has one value. KeyFields []Field // ValueFields is a list of fields that are not part of the primary key of the table. + // It can be empty in the case where all fields are part of the primary key. ValueFields []Field // RetainDeletions is a flag that indicates whether the indexer should retain From 7fd604f7582fe8f6422fc23141993e6cd0888fc9 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 14:16:53 -0400 Subject: [PATCH 09/63] update table and entity language to object --- indexer/base/entity.go | 41 ------------------ indexer/base/enum.go | 10 +++++ indexer/base/field.go | 11 +---- indexer/base/kind.go | 2 +- indexer/base/listener.go | 12 +++--- indexer/base/module_schema.go | 4 +- .../base/{table.go => object_descriptor.go} | 12 +++--- indexer/base/object_update.go | 42 +++++++++++++++++++ 8 files changed, 68 insertions(+), 66 deletions(-) delete mode 100644 indexer/base/entity.go create mode 100644 indexer/base/enum.go rename indexer/base/{table.go => object_descriptor.go} (81%) create mode 100644 indexer/base/object_update.go diff --git a/indexer/base/entity.go b/indexer/base/entity.go deleted file mode 100644 index df2ee2d3dd2f..000000000000 --- a/indexer/base/entity.go +++ /dev/null @@ -1,41 +0,0 @@ -package indexerbase - -// EntityUpdate represents an update operation on an entity in the schema. -type EntityUpdate struct { - // TableName is the name of the table that the entity belongs to in the schema. - TableName string - - // Key returns the value of the primary key of the entity and must conform to these constraints with respect - // that the schema that is defined for the entity: - // - if key represents a single column, then the value must be valid for the first column in that - // column list. For instance, if there is one column in the key of type String, then the value must be of - // type string - // - if key represents multiple columns, then the value must be a slice of values where each value is valid - // for the corresponding column in the column list. For instance, if there are two columns in the key of - // type String, String, then the value must be a slice of two strings. - // If the key has no columns, meaning that this is a singleton entity, then this value is ignored and can be nil. - Key any - - // Value returns the non-primary key columns of the entity and can either conform to the same constraints - // as EntityUpdate.Key or it may be and instance of ValueUpdates. ValueUpdates can be used as a performance - // optimization to avoid copying the values of the entity into the update and/or to omit unchanged columns. - // If this is a delete operation, then this value is ignored and can be nil. - Value any - - // Delete is a flag that indicates whether this update is a delete operation. If true, then the Value field - // is ignored and can be nil. - Delete bool -} - -// ValueUpdates is an interface that represents the value columns of an entity update. Columns that -// were not updated may be excluded from the update. Consumers should be aware that implementations -// may not filter out columns that were unchanged. However, if a column is omitted from the update -// it should be considered unchanged. -type ValueUpdates interface { - - // Iterate iterates over the columns and values in the entity update. The function should return - // true to continue iteration or false to stop iteration. Each column value should conform - // to the requirements of that column's type in the schema. Iterate returns an error if - // it was unable to decode the values properly (which could be the case in lazy evaluation). - Iterate(func(col string, value any) bool) error -} diff --git a/indexer/base/enum.go b/indexer/base/enum.go new file mode 100644 index 000000000000..b1275d8a7a95 --- /dev/null +++ b/indexer/base/enum.go @@ -0,0 +1,10 @@ +package indexerbase + +// EnumDefinition represents the definition of an enum type. +type EnumDefinition struct { + // Name is the name of the enum type. + Name string + + // Values is a list of distinct values that are part of the enum type. + Values []string +} diff --git a/indexer/base/field.go b/indexer/base/field.go index 93755f47e6c4..825fd76326d3 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -1,6 +1,6 @@ package indexerbase -// Field represents a field in a table schema. +// Field represents a field in an object descriptor. type Field struct { // Name is the name of the field. Name string @@ -17,12 +17,3 @@ type Field struct { // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. EnumDefinition EnumDefinition } - -// EnumDefinition represents the definition of an enum type. -type EnumDefinition struct { - // Name is the name of the enum type. - Name string - - // Values is a list of distinct values that are part of the enum type. - Values []string -} diff --git a/indexer/base/kind.go b/indexer/base/kind.go index ba4046ba330b..39e1e8db3e07 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -1,6 +1,6 @@ package indexerbase -// Kind represents the basic type of a field in the table schema. +// Kind represents the basic type of a field in an object descriptor. // Each kind defines the types of go values which should be accepted // by listeners and generated by decoders when providing entity updates. type Kind int diff --git a/indexer/base/listener.go b/indexer/base/listener.go index 43a66f450e76..094c3218af53 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -8,7 +8,7 @@ import ( // It is valid for any of the methods to be nil, in which case the listener will not be called for that event. // Listeners should understand the guarantees that are provided by the source they are listening to and // understand which methods will or will not be called. For instance, most blockchains will not do logical -// decoding of data out of the box, so the InitializeModuleSchema and OnEntityUpdate methods will not be called. +// decoding of data out of the box, so the InitializeModuleSchema and OnObjectUpdate methods will not be called. // These methods will only be called when listening logical decoding is setup. type Listener struct { // Initialize is called when the listener is initialized before any other methods are called. @@ -41,18 +41,18 @@ type Listener struct { // InitializeModuleSchema should be called whenever the blockchain process starts OR whenever // logical decoding of a module is initiated. An indexer listening to this event // should ensure that they have performed whatever initialization steps (such as database - // migrations) required to receive OnEntityUpdate events for the given module. If the + // migrations) required to receive OnObjectUpdate events for the given module. If the // indexer's schema is incompatible with the module's on-chain schema, the listener should return // an error. InitializeModuleSchema func(module string, schema ModuleSchema) error - // OnEntityUpdate is called whenever an entity is updated in the module. This is only called + // OnObjectUpdate is called whenever an object is updated in a module's state. This is only called // when logical data is available. It should be assumed that the same data in raw form // is also passed to OnKVPair. - OnEntityUpdate func(module string, update EntityUpdate) error + OnObjectUpdate func(module string, update ObjectUpdate) error - // CommitCatchupSync is called after all existing entities for a module have been passed to - // OnEntityUpdate during a catch-up sync which has been initiated by return -1 for lastBlock + // CommitCatchupSync is called after all existing state for a module has been passed to + // OnObjectUpdate during a catch-up sync which has been initiated by return -1 for lastBlock // in InitializeModuleSchema. The listener should commit all the data that has been received at // this point and also save the block number as the last block that has been processed so // that processing of regular block data can resume from this point in the future. diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go index 4e8b81c2be3c..7cf95fb04d4a 100644 --- a/indexer/base/module_schema.go +++ b/indexer/base/module_schema.go @@ -3,6 +3,6 @@ package indexerbase // ModuleSchema represents the logical schema of a module for purposes of indexing and querying. type ModuleSchema struct { - // Tables is a list of tables that are part of the schema for the module. - Tables []Table + // Objects is a list of objects that are part of the schema for the module. + Objects []ObjectDescriptor } diff --git a/indexer/base/table.go b/indexer/base/object_descriptor.go similarity index 81% rename from indexer/base/table.go rename to indexer/base/object_descriptor.go index f2061005cd9a..fecb8a8a0fa2 100644 --- a/indexer/base/table.go +++ b/indexer/base/object_descriptor.go @@ -1,16 +1,16 @@ package indexerbase -// Table represents a table in the schema of a module. -type Table struct { - // Name is the name of the table. +// ObjectDescriptor describes an object in the schema of a module. +type ObjectDescriptor struct { + // Name is the name of the object. Name string - // KeyFields is a list of fields that make up the primary key of the table. - // It can be empty in which case indexers should assume that this table is + // KeyFields is a list of fields that make up the primary key of the object. + // It can be empty in which case indexers should assume that this object is // a singleton and ony has one value. KeyFields []Field - // ValueFields is a list of fields that are not part of the primary key of the table. + // ValueFields is a list of fields that are not part of the primary key of the object. // It can be empty in the case where all fields are part of the primary key. ValueFields []Field diff --git a/indexer/base/object_update.go b/indexer/base/object_update.go new file mode 100644 index 000000000000..464cbf8c445d --- /dev/null +++ b/indexer/base/object_update.go @@ -0,0 +1,42 @@ +package indexerbase + +// ObjectUpdate represents an update operation on an object in a module's state. +type ObjectUpdate struct { + + // ObjectName is the name of the object type in the module's schema. + ObjectName string + + // Key returns the value of the primary key of the object and must conform to these constraints with respect + // that the schema that is defined for the object: + // - if key represents a single field, then the value must be valid for the first field in that + // field list. For instance, if there is one field in the key of type String, then the value must be of + // type string + // - if key represents multiple fields, then the value must be a slice of values where each value is valid + // for the corresponding field in the field list. For instance, if there are two fields in the key of + // type String, String, then the value must be a slice of two strings. + // If the key has no fields, meaning that this is a singleton object, then this value is ignored and can be nil. + Key any + + // Value returns the non-primary key fields of the object and can either conform to the same constraints + // as ObjectUpdate.Key or it may be and instance of ValueUpdates. ValueUpdates can be used as a performance + // optimization to avoid copying the values of the object into the update and/or to omit unchanged fields. + // If this is a delete operation, then this value is ignored and can be nil. + Value any + + // Delete is a flag that indicates whether this update is a delete operation. If true, then the Value field + // is ignored and can be nil. + Delete bool +} + +// ValueUpdates is an interface that represents the value fields of an object update. fields that +// were not updated may be excluded from the update. Consumers should be aware that implementations +// may not filter out fields that were unchanged. However, if a field is omitted from the update +// it should be considered unchanged. +type ValueUpdates interface { + + // Iterate iterates over the fields and values in the object update. The function should return + // true to continue iteration or false to stop iteration. Each field value should conform + // to the requirements of that field's type in the schema. Iterate returns an error if + // it was unable to decode the values properly (which could be the case in lazy evaluation). + Iterate(func(col string, value any) bool) error +} From 4a00094d141bd9eaaccd3b4d68d1adc10b8d691b Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 14:37:53 -0400 Subject: [PATCH 10/63] rename to type --- indexer/base/field.go | 2 +- indexer/base/kind.go | 2 +- indexer/base/module_schema.go | 4 ++-- indexer/base/{object_descriptor.go => object_type.go} | 4 ++-- indexer/base/object_update.go | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) rename indexer/base/{object_descriptor.go => object_type.go} (90%) diff --git a/indexer/base/field.go b/indexer/base/field.go index 825fd76326d3..713abdeb4c86 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -1,6 +1,6 @@ package indexerbase -// Field represents a field in an object descriptor. +// Field represents a field in an object type. type Field struct { // Name is the name of the field. Name string diff --git a/indexer/base/kind.go b/indexer/base/kind.go index 39e1e8db3e07..064a6543beaf 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -1,6 +1,6 @@ package indexerbase -// Kind represents the basic type of a field in an object descriptor. +// Kind represents the basic type of a field in an object. // Each kind defines the types of go values which should be accepted // by listeners and generated by decoders when providing entity updates. type Kind int diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go index 7cf95fb04d4a..45da535e87e3 100644 --- a/indexer/base/module_schema.go +++ b/indexer/base/module_schema.go @@ -3,6 +3,6 @@ package indexerbase // ModuleSchema represents the logical schema of a module for purposes of indexing and querying. type ModuleSchema struct { - // Objects is a list of objects that are part of the schema for the module. - Objects []ObjectDescriptor + // ObjectTypes describe the types of objects that are part of the module's schema. + ObjectTypes []ObjectType } diff --git a/indexer/base/object_descriptor.go b/indexer/base/object_type.go similarity index 90% rename from indexer/base/object_descriptor.go rename to indexer/base/object_type.go index fecb8a8a0fa2..06f8adbc92a7 100644 --- a/indexer/base/object_descriptor.go +++ b/indexer/base/object_type.go @@ -1,7 +1,7 @@ package indexerbase -// ObjectDescriptor describes an object in the schema of a module. -type ObjectDescriptor struct { +// ObjectType describes an object type a module schema. +type ObjectType struct { // Name is the name of the object. Name string diff --git a/indexer/base/object_update.go b/indexer/base/object_update.go index 464cbf8c445d..fa3d37c8e71b 100644 --- a/indexer/base/object_update.go +++ b/indexer/base/object_update.go @@ -3,8 +3,8 @@ package indexerbase // ObjectUpdate represents an update operation on an object in a module's state. type ObjectUpdate struct { - // ObjectName is the name of the object type in the module's schema. - ObjectName string + // TypeName is the name of the object type in the module's schema. + TypeName string // Key returns the value of the primary key of the object and must conform to these constraints with respect // that the schema that is defined for the object: From 0c7f52977a2fb596f7eca12ea98dc52f517b3775 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 14:50:35 -0400 Subject: [PATCH 11/63] add CHANGELOG.md --- indexer/base/CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 indexer/base/CHANGELOG.md diff --git a/indexer/base/CHANGELOG.md b/indexer/base/CHANGELOG.md new file mode 100644 index 000000000000..0c3c9d03857f --- /dev/null +++ b/indexer/base/CHANGELOG.md @@ -0,0 +1,37 @@ + + +# Changelog + +## [Unreleased] From 408ddc465115851c7d8e05507558b9ba045e4136 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 13 Jun 2024 18:44:14 -0400 Subject: [PATCH 12/63] add DecodableModule interface --- indexer/base/decoder.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 indexer/base/decoder.go diff --git a/indexer/base/decoder.go b/indexer/base/decoder.go new file mode 100644 index 000000000000..fb745c2ac6fe --- /dev/null +++ b/indexer/base/decoder.go @@ -0,0 +1,27 @@ +package indexerbase + +// DecodableModule is an interface that modules can implement to provide a ModuleDecoder. +// Usually these modules would also implement appmodule.AppModule, but that is not included +// to keep this package free of any dependencies. +type DecodableModule interface { + + // ModuleDecoder returns a ModuleDecoder for the module. + ModuleDecoder() (ModuleDecoder, error) +} + +// ModuleDecoder is a struct that contains the schema and a KVDecoder for a module. +type ModuleDecoder struct { + // Schema is the schema for the module. + Schema ModuleSchema + + // KVDecoder is a function that decodes a key-value pair into an ObjectUpdate. + // If modules pass logical updates directly to the engine and don't require logical decoding of raw bytes, + // then this function should be nil. + KVDecoder KVDecoder +} + +// KVDecoder is a function that decodes a key-value pair into an ObjectUpdate. +// If the KV-pair doesn't represent an object update, the function should return false +// as the second return value. Error should only be non-nil when the decoder expected +// to parse a valid update and was unable to. +type KVDecoder = func(key, value []byte) (ObjectUpdate, bool, error) From bc98756c26780181e9e737a7fe74c1b591b388c2 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 14 Jun 2024 14:21:07 -0400 Subject: [PATCH 13/63] make compatible with go 1.12 --- indexer/base/go.mod | 7 ++++--- indexer/base/object_update.go | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/indexer/base/go.mod b/indexer/base/go.mod index c369648761e8..a373d4dc8377 100644 --- a/indexer/base/go.mod +++ b/indexer/base/go.mod @@ -1,6 +1,7 @@ module cosmossdk.io/indexer/base -// NOTE: this go.mod should have zero dependencies and remain on an older version of Go -// to be compatible with legacy codebases. +// NOTE: this go.mod should have zero dependencies and remain on go 1.12 to stay compatible +// with all known production releases of the Cosmos SDK. This is to ensure that all historical +// apps could be patched to support indexing if desired. -go 1.19 +go 1.12 diff --git a/indexer/base/object_update.go b/indexer/base/object_update.go index fa3d37c8e71b..e99d625404fb 100644 --- a/indexer/base/object_update.go +++ b/indexer/base/object_update.go @@ -15,13 +15,13 @@ type ObjectUpdate struct { // for the corresponding field in the field list. For instance, if there are two fields in the key of // type String, String, then the value must be a slice of two strings. // If the key has no fields, meaning that this is a singleton object, then this value is ignored and can be nil. - Key any + Key interface{} // Value returns the non-primary key fields of the object and can either conform to the same constraints // as ObjectUpdate.Key or it may be and instance of ValueUpdates. ValueUpdates can be used as a performance // optimization to avoid copying the values of the object into the update and/or to omit unchanged fields. // If this is a delete operation, then this value is ignored and can be nil. - Value any + Value interface{} // Delete is a flag that indicates whether this update is a delete operation. If true, then the Value field // is ignored and can be nil. @@ -38,5 +38,5 @@ type ValueUpdates interface { // true to continue iteration or false to stop iteration. Each field value should conform // to the requirements of that field's type in the schema. Iterate returns an error if // it was unable to decode the values properly (which could be the case in lazy evaluation). - Iterate(func(col string, value any) bool) error + Iterate(func(col string, value interface{}) bool) error } From 68d0afc5ac85399f09d6b983499772dd5cadaf8f Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 08:58:09 -0400 Subject: [PATCH 14/63] remove CommitCatchupSync - catch-up design in flux, may be premature to specify this --- indexer/base/listener.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/indexer/base/listener.go b/indexer/base/listener.go index 094c3218af53..488ed87d6bbc 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -50,13 +50,6 @@ type Listener struct { // when logical data is available. It should be assumed that the same data in raw form // is also passed to OnKVPair. OnObjectUpdate func(module string, update ObjectUpdate) error - - // CommitCatchupSync is called after all existing state for a module has been passed to - // OnObjectUpdate during a catch-up sync which has been initiated by return -1 for lastBlock - // in InitializeModuleSchema. The listener should commit all the data that has been received at - // this point and also save the block number as the last block that has been processed so - // that processing of regular block data can resume from this point in the future. - CommitCatchupSync func(module string, block uint64) error } // InitializationData represents initialization data that is passed to a listener. From 50a8c37f7aa52650c8f0d408c8b390b937edd82d Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 11:51:29 -0400 Subject: [PATCH 15/63] restore validation code --- indexer/base/field.go | 123 ++++++++++++++++ indexer/base/kind.go | 227 +++++++++++++++++++++++++++++ indexer/base/kind_test.go | 290 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 640 insertions(+) create mode 100644 indexer/base/kind_test.go diff --git a/indexer/base/field.go b/indexer/base/field.go index 713abdeb4c86..fe2fad7105ab 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -1,5 +1,7 @@ package indexerbase +import "fmt" + // Field represents a field in an object type. type Field struct { // Name is the name of the field. @@ -17,3 +19,124 @@ type Field struct { // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. EnumDefinition EnumDefinition } + +// Validate validates the field. +func (c Field) Validate() error { + // non-empty name + if c.Name == "" { + return fmt.Errorf("field name cannot be empty") + } + + // valid kind + if err := c.Kind.Validate(); err != nil { + return fmt.Errorf("invalid field type for %q: %w", c.Name, err) + } + + // address prefix only valid with Bech32AddressKind + if c.Kind == Bech32AddressKind && c.AddressPrefix == "" { + return fmt.Errorf("missing address prefix for field %q", c.Name) + } else if c.Kind != Bech32AddressKind && c.AddressPrefix != "" { + return fmt.Errorf("address prefix is only valid for field %q with type Bech32AddressKind", c.Name) + } + + // enum definition only valid with EnumKind + if c.Kind == EnumKind { + if err := c.EnumDefinition.Validate(); err != nil { + return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err) + } + } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { + return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name) + } + + return nil +} + +// Validate validates the enum definition. +func (e EnumDefinition) Validate() error { + if e.Name == "" { + return fmt.Errorf("enum definition name cannot be empty") + } + if len(e.Values) == 0 { + return fmt.Errorf("enum definition values cannot be empty") + } + seen := make(map[string]bool, len(e.Values)) + for i, v := range e.Values { + if v == "" { + return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name) + } + if seen[v] { + return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name) + } + seen[v] = true + } + return nil +} + +// ValidateValue validates that the value conforms to the field's kind and nullability. +// It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind +// values are valid for their respective types behind conforming to the correct go type. +func (c Field) ValidateValue(value any) error { + if value == nil { + if !c.Nullable { + return fmt.Errorf("field %q cannot be null", c.Name) + } + return nil + } + return c.Kind.ValidateValueType(value) +} + +// ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. +// See EntityUpdate.Key for documentation on the requirements of such values. +func ValidateKey(fields []Field, value any) error { + if len(fields) == 0 { + return nil + } + + if len(fields) == 1 { + return fields[0].ValidateValue(value) + } + + values, ok := value.([]any) + if !ok { + return fmt.Errorf("expected slice of values for key fields, got %T", value) + } + + if len(fields) != len(values) { + return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]any))) + } + for i, field := range fields { + if err := field.ValidateValue(values[i]); err != nil { + return fmt.Errorf("invalid value for key field %q: %w", field.Name, err) + } + } + return nil +} + +// ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate. +// See EntityUpdate.Value for documentation on the requirements of such values. +func ValidateValue(fields []Field, value any) error { + valueUpdates, ok := value.(ValueUpdates) + if ok { + fieldMap := map[string]Field{} + for _, field := range fields { + fieldMap[field.Name] = field + } + var errs []error + valueUpdates.Iterate(func(fieldName string, value any) bool { + field, ok := fieldMap[fieldName] + if !ok { + errs = append(errs, fmt.Errorf("unknown field %q in value updates", fieldName)) + } + if err := field.ValidateValue(value); err != nil { + errs = append(errs, fmt.Errorf("invalid value for field %q: %w", fieldName, err)) + } + return true + }) + if len(errs) > 0 { + return fmt.Errorf("validation errors: %v", errs) + } + return nil + } else { + return ValidateKey(fields, value) + } +} diff --git a/indexer/base/kind.go b/indexer/base/kind.go index 064a6543beaf..af772580210a 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -1,5 +1,11 @@ package indexerbase +import ( + "encoding/json" + "fmt" + "time" +) + // Kind represents the basic type of a field in an object. // Each kind defines the types of go values which should be accepted // by listeners and generated by decoders when providing entity updates. @@ -80,3 +86,224 @@ const ( // or any type that can be marshaled to JSON using json.Marshal. JSONKind ) + +// Validate returns an error if the kind is invalid. +func (t Kind) Validate() error { + if t <= InvalidKind { + return fmt.Errorf("unknown type: %d", t) + } + if t > JSONKind { + return fmt.Errorf("invalid type: %d", t) + } + return nil +} + +// ValidateValueType returns an error if the value does not the type go type specified by the kind. +// Some fields may accept nil values, however, this method does not have any notion of +// nullability. It only checks that the value is of the correct type. +// It also doesn't perform any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind +// values are valid for their respective types. +func (t Kind) ValidateValueType(value any) error { + switch t { + case StringKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case BytesKind: + _, ok := value.([]byte) + if !ok { + return fmt.Errorf("expected []byte, got %T", value) + } + case Int8Kind: + _, ok := value.(int8) + if !ok { + return fmt.Errorf("expected int8, got %T", value) + } + case Uint8Kind: + _, ok := value.(uint8) + if !ok { + return fmt.Errorf("expected uint8, got %T", value) + } + case Int16Kind: + _, ok := value.(int16) + if !ok { + return fmt.Errorf("expected int16, got %T", value) + } + case Uint16Kind: + _, ok := value.(uint16) + if !ok { + return fmt.Errorf("expected uint16, got %T", value) + } + case Int32Kind: + _, ok := value.(int32) + if !ok { + return fmt.Errorf("expected int32, got %T", value) + } + case Uint32Kind: + _, ok := value.(uint32) + if !ok { + return fmt.Errorf("expected uint32, got %T", value) + } + case Int64Kind: + _, ok := value.(int64) + if !ok { + return fmt.Errorf("expected int64, got %T", value) + } + case Uint64Kind: + _, ok := value.(uint64) + if !ok { + return fmt.Errorf("expected uint64, got %T", value) + } + case IntegerKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + _, ok3 := value.(int64) + if !ok && !ok2 && !ok3 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case DecimalKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case BoolKind: + _, ok := value.(bool) + if !ok { + return fmt.Errorf("expected bool, got %T", value) + } + case TimeKind: + _, ok := value.(time.Time) + if !ok { + return fmt.Errorf("expected time.Time, got %T", value) + } + case DurationKind: + _, ok := value.(time.Duration) + if !ok { + return fmt.Errorf("expected time.Duration, got %T", value) + } + case Float32Kind: + _, ok := value.(float32) + if !ok { + return fmt.Errorf("expected float32, got %T", value) + } + case Float64Kind: + _, ok := value.(float64) + if !ok { + return fmt.Errorf("expected float64, got %T", value) + } + case Bech32AddressKind: + _, ok := value.(string) + _, ok2 := value.([]byte) + _, ok3 := value.(fmt.Stringer) + if !ok && !ok2 && !ok3 { + return fmt.Errorf("expected string or []byte, got %T", value) + } + case EnumKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case JSONKind: + return nil + default: + return fmt.Errorf("invalid type: %d", t) + } + return nil +} + +// String returns a string representation of the kind. +func (t Kind) String() string { + switch t { + case StringKind: + return "string" + case BytesKind: + return "bytes" + case Int8Kind: + return "int8" + case Uint8Kind: + return "uint8" + case Int16Kind: + return "int16" + case Uint16Kind: + return "uint16" + case Int32Kind: + return "int32" + case Uint32Kind: + return "uint32" + case Int64Kind: + return "int64" + case Uint64Kind: + return "uint64" + case DecimalKind: + return "decimal" + case IntegerKind: + return "integer" + case BoolKind: + return "bool" + case TimeKind: + return "time" + case DurationKind: + return "duration" + case Float32Kind: + return "float32" + case Float64Kind: + return "float64" + case Bech32AddressKind: + return "bech32address" + case EnumKind: + return "enum" + case JSONKind: + return "json" + default: + return "" + } +} + +// KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, +// return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be +// represented as strings. Generally all values which do not have a more specific type will +// return JSONKind because the framework cannot decide at this point whether the value +// can or cannot be marshaled to JSON. This method should generally only be used as a fallback +// when the kind of a field is not specified more specifically. +func KindForGoValue(value any) Kind { + switch value.(type) { + case string, fmt.Stringer: + return StringKind + case []byte: + return BytesKind + case int8: + return Int8Kind + case uint8: + return Uint8Kind + case int16: + return Int16Kind + case uint16: + return Uint16Kind + case int32: + return Int32Kind + case uint32: + return Uint32Kind + case int64: + return Int64Kind + case uint64: + return Uint64Kind + case float32: + return Float32Kind + case float64: + return Float64Kind + case bool: + return BoolKind + case time.Time: + return TimeKind + case time.Duration: + return DurationKind + case json.RawMessage: + return JSONKind + default: + return JSONKind + } +} diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go new file mode 100644 index 000000000000..18075535b067 --- /dev/null +++ b/indexer/base/kind_test.go @@ -0,0 +1,290 @@ +package indexerbase + +import ( + "strings" + "testing" + "time" +) + +func TestKind_Validate(t *testing.T) { + validKinds := []Kind{ + StringKind, + BytesKind, + Int8Kind, + Uint8Kind, + Int16Kind, + Uint16Kind, + Int32Kind, + Uint32Kind, + Int64Kind, + Uint64Kind, + IntegerKind, + DecimalKind, + BoolKind, + EnumKind, + Bech32AddressKind, + } + + for _, kind := range validKinds { + if err := kind.Validate(); err != nil { + t.Errorf("expected valid kind %s to pass validation, got: %v", kind, err) + } + } + + invalidKinds := []Kind{ + Kind(-1), + InvalidKind, + Kind(100), + } + + for _, kind := range invalidKinds { + if err := kind.Validate(); err == nil { + t.Errorf("expected invalid kind %s to fail validation, got: %v", kind, err) + } + } +} + +func TestKind_ValidateValue(t *testing.T) { + tests := []struct { + kind Kind + value any + valid bool + }{ + { + kind: StringKind, + value: "hello", + valid: true, + }, + { + kind: StringKind, + value: &strings.Builder{}, + valid: true, + }, + { + kind: StringKind, + value: []byte("hello"), + valid: false, + }, + { + kind: BytesKind, + value: []byte("hello"), + valid: true, + }, + { + kind: BytesKind, + value: "hello", + valid: false, + }, + { + kind: Int8Kind, + value: int8(1), + valid: true, + }, + { + kind: Int8Kind, + value: int16(1), + valid: false, + }, + { + kind: Uint8Kind, + value: uint8(1), + valid: true, + }, + { + kind: Uint8Kind, + value: uint16(1), + valid: false, + }, + { + kind: Int16Kind, + value: int16(1), + valid: true, + }, + { + kind: Int16Kind, + value: int32(1), + valid: false, + }, + { + kind: Uint16Kind, + value: uint16(1), + valid: true, + }, + { + kind: Uint16Kind, + value: uint32(1), + valid: false, + }, + { + kind: Int32Kind, + value: int32(1), + valid: true, + }, + { + kind: Int32Kind, + value: int64(1), + valid: false, + }, + { + kind: Uint32Kind, + value: uint32(1), + valid: true, + }, + { + kind: Uint32Kind, + value: uint64(1), + valid: false, + }, + { + kind: Int64Kind, + value: int64(1), + valid: true, + }, + { + kind: Int64Kind, + value: int32(1), + valid: false, + }, + { + kind: Uint64Kind, + value: uint64(1), + valid: true, + }, + { + kind: Uint64Kind, + value: uint32(1), + valid: false, + }, + { + kind: IntegerKind, + value: "1", + valid: true, + }, + //{ + // kind: IntegerKind, + // value: (&strings.Builder{}).WriteString("1"), + // valid: true, + //}, + { + kind: IntegerKind, + value: int32(1), + valid: false, + }, + { + kind: IntegerKind, + value: int64(1), + valid: true, + }, + { + kind: DecimalKind, + value: "1.0", + valid: true, + }, + { + kind: DecimalKind, + value: "1", + valid: true, + }, + { + kind: DecimalKind, + value: "1.1e4", + valid: true, + }, + //{ + // kind: DecimalKind, + // value: (&strings.Builder{}).WriteString("1.0"), + // valid: true, + //}, + { + kind: DecimalKind, + value: int32(1), + valid: false, + }, + { + kind: Bech32AddressKind, + value: "cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h", + valid: true, + }, + //{ + // kind: Bech32AddressKind, + // value: (&strings.Builder{}).WriteString("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), + // valid: true, + //}, + { + kind: Bech32AddressKind, + value: 1, + valid: false, + }, + { + kind: BoolKind, + value: true, + valid: true, + }, + { + kind: BoolKind, + value: false, + valid: true, + }, + { + kind: BoolKind, + value: 1, + valid: false, + }, + { + kind: EnumKind, + value: "hello", + valid: true, + }, + //{ + // kind: EnumKind, + // value: (&strings.Builder{}).WriteString("hello"), + // valid: true, + //}, + { + kind: EnumKind, + value: 1, + valid: false, + }, + { + kind: TimeKind, + value: time.Now(), + valid: true, + }, + { + kind: TimeKind, + value: "hello", + valid: false, + }, + { + kind: DurationKind, + value: time.Second, + valid: true, + }, + { + kind: DurationKind, + value: "hello", + valid: false, + }, + { + kind: Float32Kind, + value: float32(1.0), + valid: true, + }, + { + kind: Float32Kind, + value: float64(1.0), + valid: false, + }, + // TODO float64, json + } + + for i, tt := range tests { + err := tt.kind.ValidateValueType(tt.value) + if tt.valid && err != nil { + t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) + } + if !tt.valid && err == nil { + t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) + } + } +} From 6bb9c48ac5e0c934bdb20d0d3883d0d28be4b4f9 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 13:29:53 -0400 Subject: [PATCH 16/63] WIP on mermaid --- indexer/base/README.md | 47 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/indexer/base/README.md b/indexer/base/README.md index 0b96a27dc63c..17eb9115d636 100644 --- a/indexer/base/README.md +++ b/indexer/base/README.md @@ -2,4 +2,49 @@ The indexer base module is designed to provide a stable, zero-dependency base layer for the built-in indexer functionality. Packages that integrate with the indexer should feel free to depend on this package without fear of any external dependencies being pulled in. -The basic types for specifying index sources, targets and decoders are provided here along with a basic engine that ties these together. A package wishing to be an indexing source could accept an instance of `Engine` directly to be compatible with indexing. A package wishing to be a decoder can use the `Entity` and `Table` types. A package defining an indexing target should implement the `Indexer` interface. \ No newline at end of file +The basic types for specifying index sources, targets and decoders are provided here. An indexing source should accept a `Listener` instance and invoke the provided callbacks in the correct order. An indexer should provide a `Listener` instance and perform indexing operations based on the data passed to its callbacks. A module that exposes logical updates in the form of `ObjectUpdate`s should implement the `IndexableModule` interface. + +## `Listener` Callback Order + +`Listener` callbacks should be called in this order + +```mermaid +sequenceDiagram + actor Source + participant Indexer + Source ->> Indexer: Initialize + opt Have Logical Decoding + Source -->> Indexer: InitializeModuleSchema + end + loop Block + Source ->> Indexer: StartBlock + Source ->> Indexer: OnBlockHeader + Source ->> Indexer: OnTx + Source ->> Indexer: OnEvent + Source ->> Indexer: OnKVPair + Source -->> Indexer: OnObjectUpdate + Source ->> Indexer: Commit + end +``` + +Sources will generally only call `InitializeModuleSchema` and `OnObjectUpdate` if they have native logical decoding capabilities. Usually, the indexer framework will provide this functionality based on `OnKVPair` data and `IndexableModule` implementations. + +## `ModuleSchema`, `ObjectType` and `ObjectUpdate`s + +```mermaid +classDiagram + class ModuleSchema + ModuleSchema --* ObjectType + + class ObjectType { + Name string + RetainDeletions bool + } + ObjectType --* Field: KeyFields + ObjectType --* Field: ValueFields + + class Field { + Name string + Kind Kind + } +``` \ No newline at end of file From 4d6e54d81ffae9d692c74966e9c774ab3f76cab9 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 13:33:32 -0400 Subject: [PATCH 17/63] mermaid sequence diagram --- indexer/base/README.md | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/indexer/base/README.md b/indexer/base/README.md index 17eb9115d636..ee5652121b55 100644 --- a/indexer/base/README.md +++ b/indexer/base/README.md @@ -28,23 +28,3 @@ sequenceDiagram ``` Sources will generally only call `InitializeModuleSchema` and `OnObjectUpdate` if they have native logical decoding capabilities. Usually, the indexer framework will provide this functionality based on `OnKVPair` data and `IndexableModule` implementations. - -## `ModuleSchema`, `ObjectType` and `ObjectUpdate`s - -```mermaid -classDiagram - class ModuleSchema - ModuleSchema --* ObjectType - - class ObjectType { - Name string - RetainDeletions bool - } - ObjectType --* Field: KeyFields - ObjectType --* Field: ValueFields - - class Field { - Name string - Kind Kind - } -``` \ No newline at end of file From fcaa9b9ba1fa8e87c6b3c73129e37685909873e9 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 13:39:56 -0400 Subject: [PATCH 18/63] update listener docs --- indexer/base/README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/indexer/base/README.md b/indexer/base/README.md index ee5652121b55..d737ef601d52 100644 --- a/indexer/base/README.md +++ b/indexer/base/README.md @@ -13,18 +13,22 @@ sequenceDiagram actor Source participant Indexer Source ->> Indexer: Initialize - opt Have Logical Decoding - Source -->> Indexer: InitializeModuleSchema - end + Source -->> Indexer: InitializeModuleSchema loop Block Source ->> Indexer: StartBlock Source ->> Indexer: OnBlockHeader - Source ->> Indexer: OnTx - Source ->> Indexer: OnEvent - Source ->> Indexer: OnKVPair + Source -->> Indexer: OnTx + Source -->> Indexer: OnEvent + Source -->> Indexer: OnKVPair Source -->> Indexer: OnObjectUpdate Source ->> Indexer: Commit end ``` +`Initialize` should be called before anything else and get called only once.`InitializeModuleSchema` should be called at most once for every module with logical data. + Sources will generally only call `InitializeModuleSchema` and `OnObjectUpdate` if they have native logical decoding capabilities. Usually, the indexer framework will provide this functionality based on `OnKVPair` data and `IndexableModule` implementations. + +`StartBlock`, `OnBlockHeader` should be called only once at the beginning of a block and `Commit` should be called only once at the end of a block. The `OnTx`, `OnEvent`, `OnKVPair` and `OnObjectUpdate` must be called after `OnBlockHeader`, may be called multiple times within a block and indexers should not assume that the order is logical unless `InitializationData.HasEventAlignedWrites` is true. + + From 313a7782e46ba1847e0d6d93f32412ab421d9610 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 13:42:41 -0400 Subject: [PATCH 19/63] cleanup --- indexer/base/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/indexer/base/README.md b/indexer/base/README.md index d737ef601d52..3836e07395a7 100644 --- a/indexer/base/README.md +++ b/indexer/base/README.md @@ -30,5 +30,3 @@ sequenceDiagram Sources will generally only call `InitializeModuleSchema` and `OnObjectUpdate` if they have native logical decoding capabilities. Usually, the indexer framework will provide this functionality based on `OnKVPair` data and `IndexableModule` implementations. `StartBlock`, `OnBlockHeader` should be called only once at the beginning of a block and `Commit` should be called only once at the end of a block. The `OnTx`, `OnEvent`, `OnKVPair` and `OnObjectUpdate` must be called after `OnBlockHeader`, may be called multiple times within a block and indexers should not assume that the order is logical unless `InitializationData.HasEventAlignedWrites` is true. - - From acf6e526e0592d3e086be41aa5542b7123397290 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 13:56:16 -0400 Subject: [PATCH 20/63] feat(collections): support indexing --- collections/collections.go | 6 ++ collections/decoder.go | 199 +++++++++++++++++++++++++++++++++++++ collections/go.mod | 2 + 3 files changed, 207 insertions(+) create mode 100644 collections/decoder.go diff --git a/collections/collections.go b/collections/collections.go index 9de3bbc38226..cb5870d07afa 100644 --- a/collections/collections.go +++ b/collections/collections.go @@ -90,6 +90,12 @@ type Collection interface { ValueCodec() codec.UntypedValueCodec genesisHandler + + getTableSchema() indexerbase.Table + + decodeKVPair(key, value []byte, delete bool) (indexerbase.EntityUpdate, bool, error) + + isIndex() bool } // Prefix defines a segregation bytes namespace for specific collections objects. diff --git a/collections/decoder.go b/collections/decoder.go new file mode 100644 index 000000000000..c4f4d92e5223 --- /dev/null +++ b/collections/decoder.go @@ -0,0 +1,199 @@ +package collections + +import ( + "bytes" + "fmt" + "strings" + + "github.com/tidwall/btree" + + indexerbase "cosmossdk.io/indexer/base" +) + +type IndexingOptions struct { +} + +func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, error) { + decoder := moduleDecoder{ + lookup: &btree.Map[string, *collDecoder]{}, + } + + var objectTypes []indexerbase.ObjectType + for _, collName := range s.collectionsOrdered { + coll := s.collectionsByName[collName] + if coll.isIndex() { + continue + } + + schema := coll.getTableSchema() + objectTypes = append(objectTypes, schema) + decoder.lookup.Set(string(coll.GetPrefix()), &collDecoder{Collection: coll}) + } + return indexerbase.ModuleDecoder{ + Schema: indexerbase.ModuleSchema{ + ObjectTypes: objectTypes, + }, + KVDecoder: decoder.decodeKV, + }, nil +} + +type moduleDecoder struct { + lookup *btree.Map[string, *collDecoder] +} + +func (m moduleDecoder) decodeKV(key, value []byte) (indexerbase.ObjectUpdate, bool, error) { + ks := string(key) + var cd *collDecoder + m.lookup.Descend(ks, func(prefix string, cur *collDecoder) bool { + bytesPrefix := cur.GetPrefix() + if bytes.HasPrefix(key, bytesPrefix) { + cd = cur + return true + } + return false + }) + if cd == nil { + return indexerbase.ObjectUpdate{}, false, nil + } + + return cd.decodeKVPair(key, value, false) +} + +type collDecoder struct { + Collection +} + +//type moduleStateDecoder struct { +// schema Schema +// collectionsIndex btree.BTree +//} +// +//func (m moduleStateDecoder) getCollectionForKey(key []byte) Collection { +// panic("implement me") +//} +// +//func (m moduleStateDecoder) DecodeSet(key, value []byte) (indexerbase.EntityUpdate, bool, error) { +// coll := m.getCollectionForKey(key) +// if coll == nil { +// return indexerbase.EntityUpdate{}, false, nil +// } +// +// return coll.decodeKVPair(key, value) +//} +// +//func (m moduleStateDecoder) DecodeDelete(key []byte) (indexerbase.EntityDelete, bool, error) { +// coll := m.getCollectionForKey(key) +// return coll.decodeDelete(key) +//} + +func (c collectionImpl[K, V]) getTableSchema() indexerbase.ObjectType { + var keyFields []indexerbase.Field + var valueFields []indexerbase.Field + + if hasSchema, ok := c.m.kc.(IndexableCodec); ok { + keyFields = hasSchema.SchemaFields() + } else { + var k K + keyFields, _ = extractFields(k) + } + ensureNames(c.m.kc, "key", keyFields) + + if hasSchema, ok := c.m.vc.(IndexableCodec); ok { + valueFields = hasSchema.SchemaFields() + } else { + var v V + valueFields, _ = extractFields(v) + } + ensureNames(c.m.vc, "value", valueFields) + + return indexerbase.ObjectType{ + Name: c.GetName(), + KeyFields: keyFields, + ValueFields: valueFields, + } +} + +func extractFields(x any) ([]indexerbase.Field, func(any) any) { + if hasSchema, ok := x.(IndexableCodec); ok { + return hasSchema.SchemaFields(), nil + } + + ty := indexerbase.TypeForGoValue(x) + if ty > 0 { + return []indexerbase.Field{{Kind: ty}}, nil + } + + panic(fmt.Errorf("unsupported type %T", x)) +} + +func ensureNames(x any, defaultName string, cols []indexerbase.Field) { + var names []string = nil + if hasName, ok := x.(interface{ Name() string }); ok { + name := hasName.Name() + if name != "" { + names = strings.Split(hasName.Name(), ",") + } + } + for i, col := range cols { + if names != nil && i < len(names) { + col.Name = names[i] + } else { + if col.Name == "" { + if i == 0 && len(cols) == 1 { + col.Name = defaultName + } else { + col.Name = fmt.Sprintf("%s%d", defaultName, i+1) + } + } + } + cols[i] = col + } +} + +func (c collectionImpl[K, V]) decodeKVPair(key, value []byte, delete bool) (indexerbase.ObjectUpdate, bool, error) { + // strip prefix + key = key[len(c.GetPrefix()):] + var k any + var err error + if decodeAny, ok := c.m.kc.(DecodeIndexable); ok { + k, err = decodeAny.DecodeIndexable(key) + } else { + _, k, err = c.m.kc.Decode(key) + } + if err != nil { + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + }, false, err + } + + if delete { + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + Key: k, + Delete: true, + }, true, nil + } + + var v any + if decodeAny, ok := c.m.vc.(DecodeIndexable); ok { + v, err = decodeAny.DecodeIndexable(value) + } else { + v, err = c.m.vc.Decode(value) + } + if err != nil { + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + }, false, err + } + + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + Key: k, + Value: v, + }, true, nil +} + +type IndexableCodec interface { + SchemaFields() []indexerbase.Field + DecodeIndexable([]byte) (any, error) +} diff --git a/collections/go.mod b/collections/go.mod index cf803ec6f903..d8c61db0176b 100644 --- a/collections/go.mod +++ b/collections/go.mod @@ -7,6 +7,7 @@ require ( cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.9.0 pgregory.net/rapid v1.1.0 + cosmossdk.io/indexer/base v0.0.0-00010101000000-000000000000 ) require ( @@ -29,4 +30,5 @@ require ( replace ( cosmossdk.io/core => ../core cosmossdk.io/core/testing => ../core/testing + cosmossdk.io/indexer/base => ../indexer/base ) From 3bb5ef69aed12b6fd693a8985d9e2762a8b828bc Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 13:57:57 -0400 Subject: [PATCH 21/63] updates --- collections/{decoder.go => indexing.go} | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename collections/{decoder.go => indexing.go} (96%) diff --git a/collections/decoder.go b/collections/indexing.go similarity index 96% rename from collections/decoder.go rename to collections/indexing.go index c4f4d92e5223..12037a6cf130 100644 --- a/collections/decoder.go +++ b/collections/indexing.go @@ -11,6 +11,7 @@ import ( ) type IndexingOptions struct { + RetainDeletionsFor []string } func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, error) { @@ -118,7 +119,7 @@ func extractFields(x any) ([]indexerbase.Field, func(any) any) { return hasSchema.SchemaFields(), nil } - ty := indexerbase.TypeForGoValue(x) + ty := indexerbase.KindForGoValue(x) if ty > 0 { return []indexerbase.Field{{Kind: ty}}, nil } @@ -155,7 +156,7 @@ func (c collectionImpl[K, V]) decodeKVPair(key, value []byte, delete bool) (inde key = key[len(c.GetPrefix()):] var k any var err error - if decodeAny, ok := c.m.kc.(DecodeIndexable); ok { + if decodeAny, ok := c.m.kc.(IndexableCodec); ok { k, err = decodeAny.DecodeIndexable(key) } else { _, k, err = c.m.kc.Decode(key) @@ -175,7 +176,7 @@ func (c collectionImpl[K, V]) decodeKVPair(key, value []byte, delete bool) (inde } var v any - if decodeAny, ok := c.m.vc.(DecodeIndexable); ok { + if decodeAny, ok := c.m.vc.(IndexableCodec); ok { v, err = decodeAny.DecodeIndexable(value) } else { v, err = c.m.vc.Decode(value) From 3606a0417d1d250a102c6a87008e1c7e7afd886f Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 20:30:23 -0400 Subject: [PATCH 22/63] update validation --- indexer/base/field.go | 15 +++++++----- indexer/base/kind.go | 54 +++++++++++++++++++++---------------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/indexer/base/field.go b/indexer/base/field.go index fe2fad7105ab..a052d84cdfc1 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -75,7 +75,7 @@ func (e EnumDefinition) Validate() error { // ValidateValue validates that the value conforms to the field's kind and nullability. // It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind // values are valid for their respective types behind conforming to the correct go type. -func (c Field) ValidateValue(value any) error { +func (c Field) ValidateValue(value interface{}) error { if value == nil { if !c.Nullable { return fmt.Errorf("field %q cannot be null", c.Name) @@ -87,7 +87,7 @@ func (c Field) ValidateValue(value any) error { // ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. // See EntityUpdate.Key for documentation on the requirements of such values. -func ValidateKey(fields []Field, value any) error { +func ValidateKey(fields []Field, value interface{}) error { if len(fields) == 0 { return nil } @@ -96,13 +96,13 @@ func ValidateKey(fields []Field, value any) error { return fields[0].ValidateValue(value) } - values, ok := value.([]any) + values, ok := value.([]interface{}) if !ok { return fmt.Errorf("expected slice of values for key fields, got %T", value) } if len(fields) != len(values) { - return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]any))) + return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]interface{}))) } for i, field := range fields { if err := field.ValidateValue(values[i]); err != nil { @@ -114,7 +114,7 @@ func ValidateKey(fields []Field, value any) error { // ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate. // See EntityUpdate.Value for documentation on the requirements of such values. -func ValidateValue(fields []Field, value any) error { +func ValidateValue(fields []Field, value interface{}) error { valueUpdates, ok := value.(ValueUpdates) if ok { fieldMap := map[string]Field{} @@ -122,7 +122,7 @@ func ValidateValue(fields []Field, value any) error { fieldMap[field.Name] = field } var errs []error - valueUpdates.Iterate(func(fieldName string, value any) bool { + err := valueUpdates.Iterate(func(fieldName string, value interface{}) bool { field, ok := fieldMap[fieldName] if !ok { errs = append(errs, fmt.Errorf("unknown field %q in value updates", fieldName)) @@ -132,6 +132,9 @@ func ValidateValue(fields []Field, value any) error { } return true }) + if err != nil { + return err + } if len(errs) > 0 { return fmt.Errorf("validation errors: %v", errs) } diff --git a/indexer/base/kind.go b/indexer/base/kind.go index af772580210a..c567380d006e 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -103,14 +103,10 @@ func (t Kind) Validate() error { // nullability. It only checks that the value is of the correct type. // It also doesn't perform any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind // values are valid for their respective types. -func (t Kind) ValidateValueType(value any) error { +func (t Kind) ValidateValueType(value interface{}) error { switch t { case StringKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) - } + return checkStringLike(value) case BytesKind: _, ok := value.([]byte) if !ok { @@ -157,18 +153,13 @@ func (t Kind) ValidateValueType(value any) error { return fmt.Errorf("expected uint64, got %T", value) } case IntegerKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - _, ok3 := value.(int64) - if !ok && !ok2 && !ok3 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + _, ok := value.(int64) + if ok { + return nil } + return checkStringLike(value) case DecimalKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) - } + return checkStringLike(value) case BoolKind: _, ok := value.(bool) if !ok { @@ -195,18 +186,13 @@ func (t Kind) ValidateValueType(value any) error { return fmt.Errorf("expected float64, got %T", value) } case Bech32AddressKind: - _, ok := value.(string) - _, ok2 := value.([]byte) - _, ok3 := value.(fmt.Stringer) - if !ok && !ok2 && !ok3 { - return fmt.Errorf("expected string or []byte, got %T", value) + _, ok := value.([]byte) + if ok { + return nil } + return checkStringLike(value) case EnumKind: - _, ok := value.(string) - _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) - } + return checkStringLike(value) case JSONKind: return nil default: @@ -215,6 +201,20 @@ func (t Kind) ValidateValueType(value any) error { return nil } +func checkStringLike(value interface{}) error { + _, ok := value.(string) + if ok { + return nil + } + + _, ok2 := value.(fmt.Stringer) + if ok2 { + return nil + } + + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) +} + // String returns a string representation of the kind. func (t Kind) String() string { switch t { @@ -269,7 +269,7 @@ func (t Kind) String() string { // return JSONKind because the framework cannot decide at this point whether the value // can or cannot be marshaled to JSON. This method should generally only be used as a fallback // when the kind of a field is not specified more specifically. -func KindForGoValue(value any) Kind { +func KindForGoValue(value interface{}) Kind { switch value.(type) { case string, fmt.Stringer: return StringKind From 7cf96787ded216fdf1756f974224ead4f0d9ffdc Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 20:47:06 -0400 Subject: [PATCH 23/63] string tests --- indexer/base/field_test.go | 1 + indexer/base/kind.go | 2 +- indexer/base/kind_test.go | 109 +++++++++++++++++++++++++++++-------- 3 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 indexer/base/field_test.go diff --git a/indexer/base/field_test.go b/indexer/base/field_test.go new file mode 100644 index 000000000000..ef34355a0709 --- /dev/null +++ b/indexer/base/field_test.go @@ -0,0 +1 @@ +package indexerbase diff --git a/indexer/base/kind.go b/indexer/base/kind.go index c567380d006e..d1a1a7529ea2 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -259,7 +259,7 @@ func (t Kind) String() string { case JSONKind: return "json" default: - return "" + return fmt.Sprintf("invalid(%d)", t) } } diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go index 18075535b067..8fe87d81f413 100644 --- a/indexer/base/kind_test.go +++ b/indexer/base/kind_test.go @@ -1,6 +1,7 @@ package indexerbase import ( + "encoding/json" "strings" "testing" "time" @@ -47,7 +48,7 @@ func TestKind_Validate(t *testing.T) { func TestKind_ValidateValue(t *testing.T) { tests := []struct { kind Kind - value any + value interface{} valid bool }{ { @@ -57,7 +58,7 @@ func TestKind_ValidateValue(t *testing.T) { }, { kind: StringKind, - value: &strings.Builder{}, + value: stringBuilder("hello"), valid: true, }, { @@ -160,11 +161,11 @@ func TestKind_ValidateValue(t *testing.T) { value: "1", valid: true, }, - //{ - // kind: IntegerKind, - // value: (&strings.Builder{}).WriteString("1"), - // valid: true, - //}, + { + kind: IntegerKind, + value: stringBuilder("1"), + valid: true, + }, { kind: IntegerKind, value: int32(1), @@ -190,11 +191,11 @@ func TestKind_ValidateValue(t *testing.T) { value: "1.1e4", valid: true, }, - //{ - // kind: DecimalKind, - // value: (&strings.Builder{}).WriteString("1.0"), - // valid: true, - //}, + { + kind: DecimalKind, + value: stringBuilder("1.0"), + valid: true, + }, { kind: DecimalKind, value: int32(1), @@ -205,11 +206,11 @@ func TestKind_ValidateValue(t *testing.T) { value: "cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h", valid: true, }, - //{ - // kind: Bech32AddressKind, - // value: (&strings.Builder{}).WriteString("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), - // valid: true, - //}, + { + kind: Bech32AddressKind, + value: stringBuilder("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), + valid: true, + }, { kind: Bech32AddressKind, value: 1, @@ -235,11 +236,11 @@ func TestKind_ValidateValue(t *testing.T) { value: "hello", valid: true, }, - //{ - // kind: EnumKind, - // value: (&strings.Builder{}).WriteString("hello"), - // valid: true, - //}, + { + kind: EnumKind, + value: stringBuilder("hello"), + valid: true, + }, { kind: EnumKind, value: 1, @@ -275,7 +276,26 @@ func TestKind_ValidateValue(t *testing.T) { value: float64(1.0), valid: false, }, - // TODO float64, json + { + kind: Float64Kind, + value: float64(1.0), + valid: true, + }, + { + kind: Float64Kind, + value: float32(1.0), + valid: false, + }, + { + kind: JSONKind, + value: "hello", + valid: true, + }, + { + kind: JSONKind, + value: json.RawMessage("{}"), + valid: true, + }, } for i, tt := range tests { @@ -288,3 +308,46 @@ func TestKind_ValidateValue(t *testing.T) { } } } + +func stringBuilder(x string) interface{} { + b := &strings.Builder{} + _, err := b.WriteString(x) + if err != nil { + panic(err) + } + return b +} + +func TestKindString(t *testing.T) { + tests := []struct { + kind Kind + want string + }{ + {StringKind, "string"}, + {BytesKind, "bytes"}, + {Int8Kind, "int8"}, + {Uint8Kind, "uint8"}, + {Int16Kind, "int16"}, + {Uint16Kind, "uint16"}, + {Int32Kind, "int32"}, + {Uint32Kind, "uint32"}, + {Int64Kind, "int64"}, + {Uint64Kind, "uint64"}, + {IntegerKind, "integer"}, + {DecimalKind, "decimal"}, + {BoolKind, "bool"}, + {TimeKind, "time"}, + {DurationKind, "duration"}, + {Float32Kind, "float32"}, + {Float64Kind, "float64"}, + {JSONKind, "json"}, + {EnumKind, "enum"}, + {Bech32AddressKind, "bech32address"}, + {InvalidKind, "invalid(0)"}, + } + for i, tt := range tests { + if got := tt.kind.String(); got != tt.want { + t.Errorf("test %d: Kind.String() = %v, want %v", i, got, tt.want) + } + } +} From df3cde1b46f58fd0a5300946e22a4fa203d1d1d2 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 20:50:59 -0400 Subject: [PATCH 24/63] TestKindForGoValue --- indexer/base/kind.go | 4 +++- indexer/base/kind_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/indexer/base/kind.go b/indexer/base/kind.go index d1a1a7529ea2..ddaa6fec627f 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -271,7 +271,7 @@ func (t Kind) String() string { // when the kind of a field is not specified more specifically. func KindForGoValue(value interface{}) Kind { switch value.(type) { - case string, fmt.Stringer: + case string: return StringKind case []byte: return BytesKind @@ -303,6 +303,8 @@ func KindForGoValue(value interface{}) Kind { return DurationKind case json.RawMessage: return JSONKind + case fmt.Stringer: + return StringKind default: return JSONKind } diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go index 8fe87d81f413..dd3e15da7e85 100644 --- a/indexer/base/kind_test.go +++ b/indexer/base/kind_test.go @@ -351,3 +351,32 @@ func TestKindString(t *testing.T) { } } } + +func TestKindForGoValue(t *testing.T) { + tests := []struct { + value interface{} + want Kind + }{ + {"hello", StringKind}, + {stringBuilder("hello"), StringKind}, + {[]byte("hello"), BytesKind}, + {int8(1), Int8Kind}, + {uint8(1), Uint8Kind}, + {int16(1), Int16Kind}, + {uint16(1), Uint16Kind}, + {int32(1), Int32Kind}, + {uint32(1), Uint32Kind}, + {int64(1), Int64Kind}, + {uint64(1), Uint64Kind}, + {true, BoolKind}, + {time.Now(), TimeKind}, + {time.Second, DurationKind}, + {json.RawMessage("{}"), JSONKind}, + {map[string]interface{}{"a": 1}, JSONKind}, + } + for i, tt := range tests { + if got := KindForGoValue(tt.value); got != tt.want { + t.Errorf("test %d: KindForGoValue(%v) = %v, want %v", i, tt.value, got, tt.want) + } + } +} From 7c7ff793f019275b7795afa077b9463b8cf7713c Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 20:57:22 -0400 Subject: [PATCH 25/63] update validation --- indexer/base/field.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/indexer/base/field.go b/indexer/base/field.go index a052d84cdfc1..4a237abff3f0 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -72,6 +72,15 @@ func (e EnumDefinition) Validate() error { return nil } +func (e EnumDefinition) ValidateValue(value string) error { + for _, v := range e.Values { + if v == value { + return nil + } + } + return fmt.Errorf("value %q is not a valid enum value for %s", value, e.Name) +} + // ValidateValue validates that the value conforms to the field's kind and nullability. // It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind // values are valid for their respective types behind conforming to the correct go type. @@ -82,7 +91,16 @@ func (c Field) ValidateValue(value interface{}) error { } return nil } - return c.Kind.ValidateValueType(value) + err := c.Kind.ValidateValueType(value) + if err != nil { + return fmt.Errorf("invalid value for field %q: %w", c.Name, err) + } + + if c.Kind == EnumKind { + return c.EnumDefinition.ValidateValue(value.(string)) + } + + return nil } // ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. From 2eb3ed2c95ee3b7d3fa9bb07c6c3e0a53c6595d1 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 21:04:18 -0400 Subject: [PATCH 26/63] WIP on TestField_Validate --- indexer/base/field_test.go | 86 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/indexer/base/field_test.go b/indexer/base/field_test.go index ef34355a0709..c1d2e274dd7f 100644 --- a/indexer/base/field_test.go +++ b/indexer/base/field_test.go @@ -1 +1,87 @@ package indexerbase + +import "testing" + +func TestField_Validate(t *testing.T) { + tests := []struct { + name string + field Field + error string + }{ + { + name: "valid field", + field: Field{ + Name: "field1", + Kind: StringKind, + }, + error: "", + }, + { + name: "empty name", + field: Field{ + Name: "", + Kind: StringKind, + }, + error: "field name cannot be empty", + }, + { + name: "invalid kind", + field: Field{ + Name: "field1", + Kind: Kind(-1), + }, + error: "invalid field type for \"field1\": invalid type: -1", + }, + { + name: "missing address prefix", + field: Field{ + Name: "field1", + Kind: Bech32AddressKind, + }, + error: "missing address prefix for field \"field1\"", + }, + { + name: "address prefix with non-Bech32AddressKind", + field: Field{ + Name: "field1", + Kind: StringKind, + AddressPrefix: "prefix", + }, + error: "address prefix is only valid for field \"field1\" with type Bech32AddressKind", + }, + { + name: "invalid enum definition", + field: Field{ + Name: "field1", + Kind: EnumKind, + }, + error: "invalid enum definition for field \"field1\": enum definition name cannot be empty", + }, + { + name: "enum definition with non-EnumKind", + field: Field{ + Name: "field1", + Kind: StringKind, + EnumDefinition: EnumDefinition{Name: "enum"}, + }, + error: "enum definition is only valid for field \"field1\" with type EnumKind", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.field.Validate() + if tt.error == "" { + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + } else { + if err == nil { + t.Errorf("expected error, got nil") + } else if err.Error() != tt.error { + t.Errorf("expected error: %s, got: %v", tt.error, err) + } + } + }) + } +} From 8e2db24adb6ca9ee72162c67d672391a3e160acd Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 11:55:20 -0400 Subject: [PATCH 27/63] simplifications --- indexer/base/kind.go | 195 ++++++++++++++++++++++---------------- indexer/base/kind_test.go | 148 +++++++++++++++++------------ 2 files changed, 203 insertions(+), 140 deletions(-) diff --git a/indexer/base/kind.go b/indexer/base/kind.go index ddaa6fec627f..c5abf2b75a0b 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -3,6 +3,7 @@ package indexerbase import ( "encoding/json" "fmt" + "regexp" "time" ) @@ -15,8 +16,7 @@ const ( // InvalidKind indicates that an invalid type. InvalidKind Kind = iota - // StringKind is a string type and values of this type must be of the go type string - // or implement fmt.Stringer(). + // StringKind is a string type and values of this type must be of the go type string. StringKind // BytesKind is a bytes type and values of this type must be of the go type []byte. @@ -47,14 +47,14 @@ const ( Uint64Kind // IntegerKind represents an arbitrary precision integer number. Values of this type must - // be of the go type int64, string or a type that implements fmt.Stringer with the resulted string - // formatted as an integer number. + // be of the go type string and formatted as base10 integers, specifically matching to + // the IntegerFormat regex. IntegerKind // DecimalKind represents an arbitrary precision decimal or integer number. Values of this type - // must be of the go type string or a type that implements fmt.Stringer with the resulting string - // formatted as decimal numbers with an optional fractional part. Exponential E-notation - // is supported but NaN and Infinity are not. + // must be of the go type string or formatted as decimal numbers with an optional fractional part. + // Exponential e-notation is supported but NaN and Infinity are not. Values must match the + // DecimalFormat regex. DecimalKind // BoolKind is a boolean type and values of this type must be of the go type bool. @@ -72,18 +72,18 @@ const ( // Float64Kind is a float64 type and values of this type must be of the go type float64. Float64Kind - // Bech32AddressKind is a bech32 address type and values of this type must be of the go type string or []byte - // or a type which implements fmt.Stringer. Fields of this type are expected to set the AddressPrefix field - // in the field definition to the bech32 address prefix. + // Bech32AddressKind is a bech32 address type and values of this type must be of the go type []byte. + // Fields of this type are expected to set the AddressPrefix field in the field definition to the + // bech32 address prefix so that indexers can properly convert them to strings. Bech32AddressKind - // EnumKind is an enum type and values of this type must be of the go type string or implement fmt.Stringer. + // EnumKind is an enum type and values of this type must be of the go type string. // Fields of this type are expected to set the EnumDefinition field in the field definition to the enum // definition. EnumKind - // JSONKind is a JSON type and values of this type can either be of go type json.RawMessage - // or any type that can be marshaled to JSON using json.Marshal. + // JSONKind is a JSON type and values of this type should be of go type json.RawMessage and represent + // valid JSON. JSONKind ) @@ -98,15 +98,66 @@ func (t Kind) Validate() error { return nil } -// ValidateValueType returns an error if the value does not the type go type specified by the kind. +// String returns a string representation of the kind. +func (t Kind) String() string { + switch t { + case StringKind: + return "string" + case BytesKind: + return "bytes" + case Int8Kind: + return "int8" + case Uint8Kind: + return "uint8" + case Int16Kind: + return "int16" + case Uint16Kind: + return "uint16" + case Int32Kind: + return "int32" + case Uint32Kind: + return "uint32" + case Int64Kind: + return "int64" + case Uint64Kind: + return "uint64" + case DecimalKind: + return "decimal" + case IntegerKind: + return "integer" + case BoolKind: + return "bool" + case TimeKind: + return "time" + case DurationKind: + return "duration" + case Float32Kind: + return "float32" + case Float64Kind: + return "float64" + case Bech32AddressKind: + return "bech32address" + case EnumKind: + return "enum" + case JSONKind: + return "json" + default: + return fmt.Sprintf("invalid(%d)", t) + } +} + +// ValidateValueType returns an error if the value does not conform to the expected go type. // Some fields may accept nil values, however, this method does not have any notion of -// nullability. It only checks that the value is of the correct type. -// It also doesn't perform any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind -// values are valid for their respective types. +// nullability. This method only validates that the go type of the value is correct for the kind +// and does not validate string or json formats. Kind.ValidateValue does a more thorough validation +// of number and json string formatting. func (t Kind) ValidateValueType(value interface{}) error { switch t { case StringKind: - return checkStringLike(value) + _, ok := value.(string) + if !ok { + return fmt.Errorf("expected string, got %T", value) + } case BytesKind: _, ok := value.([]byte) if !ok { @@ -153,13 +204,16 @@ func (t Kind) ValidateValueType(value interface{}) error { return fmt.Errorf("expected uint64, got %T", value) } case IntegerKind: - _, ok := value.(int64) - if ok { - return nil + _, ok := value.(string) + if !ok { + return fmt.Errorf("expected string, got %T", value) } - return checkStringLike(value) + case DecimalKind: - return checkStringLike(value) + _, ok := value.(string) + if !ok { + return fmt.Errorf("expected string, got %T", value) + } case BoolKind: _, ok := value.(bool) if !ok { @@ -187,82 +241,61 @@ func (t Kind) ValidateValueType(value interface{}) error { } case Bech32AddressKind: _, ok := value.([]byte) - if ok { - return nil + if !ok { + return fmt.Errorf("expected []byte, got %T", value) } - return checkStringLike(value) case EnumKind: - return checkStringLike(value) + _, ok := value.(string) + if !ok { + return fmt.Errorf("expected string, got %T", value) + } case JSONKind: - return nil + _, ok := value.(json.RawMessage) + if !ok { + return fmt.Errorf("expected json.RawMessage, got %T", value) + } default: return fmt.Errorf("invalid type: %d", t) } return nil } -func checkStringLike(value interface{}) error { - _, ok := value.(string) - if ok { - return nil - } - - _, ok2 := value.(fmt.Stringer) - if ok2 { - return nil +// ValidateValue returns an error if the value does not conform to the expected go type and format. +// It is more thorough, but slower, than Kind.ValidateValueType and validates that Integer, Decimal and JSON +// values are formatted correctly. It cannot validate enum values because Kind's do not have enum schemas. +func (t Kind) ValidateValue(value interface{}) error { + err := t.ValidateValueType(value) + if err != nil { + return err } - return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) -} - -// String returns a string representation of the kind. -func (t Kind) String() string { switch t { - case StringKind: - return "string" - case BytesKind: - return "bytes" - case Int8Kind: - return "int8" - case Uint8Kind: - return "uint8" - case Int16Kind: - return "int16" - case Uint16Kind: - return "uint16" - case Int32Kind: - return "int32" - case Uint32Kind: - return "uint32" - case Int64Kind: - return "int64" - case Uint64Kind: - return "uint64" - case DecimalKind: - return "decimal" case IntegerKind: - return "integer" - case BoolKind: - return "bool" - case TimeKind: - return "time" - case DurationKind: - return "duration" - case Float32Kind: - return "float32" - case Float64Kind: - return "float64" - case Bech32AddressKind: - return "bech32address" - case EnumKind: - return "enum" + if !integerRegex.Match([]byte(value.(string))) { + return fmt.Errorf("expected base10 integer, got %s", value) + } + case DecimalKind: + if !decimalRegex.Match([]byte(value.(string))) { + return fmt.Errorf("expected decimal number, got %s", value) + } case JSONKind: - return "json" + if !json.Valid(value.(json.RawMessage)) { + return fmt.Errorf("expected valid JSON, got %s", value) + } default: - return fmt.Sprintf("invalid(%d)", t) + return nil } + return nil } +const ( + IntegerFormat = `^-?[0-9]+$` + DecimalFormat = `^-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$` +) + +var integerRegex = regexp.MustCompile(IntegerFormat) +var decimalRegex = regexp.MustCompile(DecimalFormat) + // KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, // return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be // represented as strings. Generally all values which do not have a more specific type will diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go index dd3e15da7e85..80121f195892 100644 --- a/indexer/base/kind_test.go +++ b/indexer/base/kind_test.go @@ -2,7 +2,7 @@ package indexerbase import ( "encoding/json" - "strings" + "fmt" "testing" "time" ) @@ -45,7 +45,7 @@ func TestKind_Validate(t *testing.T) { } } -func TestKind_ValidateValue(t *testing.T) { +func TestKind_ValidateValueType(t *testing.T) { tests := []struct { kind Kind value interface{} @@ -56,11 +56,6 @@ func TestKind_ValidateValue(t *testing.T) { value: "hello", valid: true, }, - { - kind: StringKind, - value: stringBuilder("hello"), - valid: true, - }, { kind: StringKind, value: []byte("hello"), @@ -161,21 +156,11 @@ func TestKind_ValidateValue(t *testing.T) { value: "1", valid: true, }, - { - kind: IntegerKind, - value: stringBuilder("1"), - valid: true, - }, { kind: IntegerKind, value: int32(1), valid: false, }, - { - kind: IntegerKind, - value: int64(1), - valid: true, - }, { kind: DecimalKind, value: "1.0", @@ -191,11 +176,6 @@ func TestKind_ValidateValue(t *testing.T) { value: "1.1e4", valid: true, }, - { - kind: DecimalKind, - value: stringBuilder("1.0"), - valid: true, - }, { kind: DecimalKind, value: int32(1), @@ -203,12 +183,7 @@ func TestKind_ValidateValue(t *testing.T) { }, { kind: Bech32AddressKind, - value: "cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h", - valid: true, - }, - { - kind: Bech32AddressKind, - value: stringBuilder("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), + value: []byte("hello"), valid: true, }, { @@ -236,11 +211,6 @@ func TestKind_ValidateValue(t *testing.T) { value: "hello", valid: true, }, - { - kind: EnumKind, - value: stringBuilder("hello"), - valid: true, - }, { kind: EnumKind, value: 1, @@ -286,11 +256,6 @@ func TestKind_ValidateValue(t *testing.T) { value: float32(1.0), valid: false, }, - { - kind: JSONKind, - value: "hello", - valid: true, - }, { kind: JSONKind, value: json.RawMessage("{}"), @@ -299,26 +264,88 @@ func TestKind_ValidateValue(t *testing.T) { } for i, tt := range tests { - err := tt.kind.ValidateValueType(tt.value) - if tt.valid && err != nil { - t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) - } - if !tt.valid && err == nil { - t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) - } + t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { + err := tt.kind.ValidateValueType(tt.value) + if tt.valid && err != nil { + t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) + } + if !tt.valid && err == nil { + t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) + } + }) } } -func stringBuilder(x string) interface{} { - b := &strings.Builder{} - _, err := b.WriteString(x) - if err != nil { - panic(err) +func TestKind_ValidateValue(t *testing.T) { + tests := []struct { + kind Kind + value interface{} + valid bool + }{ + // check a few basic cases + {StringKind, "hello", true}, + {Int64Kind, int64(1), true}, + // check integer, decimal and json more thoroughly + {IntegerKind, "1", true}, + {IntegerKind, "0", true}, + {IntegerKind, "10", true}, + {IntegerKind, "-100", true}, + {IntegerKind, "1.0", false}, + {IntegerKind, "00", true}, // leading zeros are allowed + {IntegerKind, "001", true}, + {IntegerKind, "-01", true}, + {IntegerKind, "", false}, + {IntegerKind, "abc", false}, + {IntegerKind, "abc100", false}, + {DecimalKind, "1.0", true}, + {DecimalKind, "0.0", true}, + {DecimalKind, "-100.075", true}, + {DecimalKind, "1002346.000", true}, + {DecimalKind, "0", true}, + {DecimalKind, "10", true}, + {DecimalKind, "-100", true}, + {DecimalKind, "1", true}, + {DecimalKind, "1.0e4", true}, + {DecimalKind, "1.0e-4", true}, + {DecimalKind, "1.0e+4", true}, + {DecimalKind, "1.0e", false}, + {DecimalKind, "1.0e4.0", false}, + {DecimalKind, "1.0e-4.0", false}, + {DecimalKind, "1.0e+4.0", false}, + {DecimalKind, "-1.0e-4", true}, + {DecimalKind, "-1.0e+4", true}, + {DecimalKind, "-1.0E4", true}, + {DecimalKind, "1E-9", true}, + {DecimalKind, "", false}, + {DecimalKind, "abc", false}, + {DecimalKind, "abc", false}, + {JSONKind, json.RawMessage(`{"a":10}`), true}, + {JSONKind, json.RawMessage("10"), true}, + {JSONKind, json.RawMessage("10.0"), true}, + {JSONKind, json.RawMessage("true"), true}, + {JSONKind, json.RawMessage("null"), true}, + {JSONKind, json.RawMessage(`"abc"`), true}, + {JSONKind, json.RawMessage(`[1,true,0.1,"abc",{"b":3}]`), true}, + {JSONKind, json.RawMessage(`"abc`), false}, + {JSONKind, json.RawMessage(`tru`), false}, + {JSONKind, json.RawMessage(`[`), false}, + {JSONKind, json.RawMessage(`{`), false}, + } + + for i, tt := range tests { + t.Run(fmt.Sprintf("test %v %s", tt.kind, tt.value), func(t *testing.T) { + err := tt.kind.ValidateValue(tt.value) + if tt.valid && err != nil { + t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) + } + if !tt.valid && err == nil { + t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) + } + }) } - return b } -func TestKindString(t *testing.T) { +func TestKind_String(t *testing.T) { tests := []struct { kind Kind want string @@ -346,9 +373,11 @@ func TestKindString(t *testing.T) { {InvalidKind, "invalid(0)"}, } for i, tt := range tests { - if got := tt.kind.String(); got != tt.want { - t.Errorf("test %d: Kind.String() = %v, want %v", i, got, tt.want) - } + t.Run(fmt.Sprintf("test %s", tt.kind), func(t *testing.T) { + if got := tt.kind.String(); got != tt.want { + t.Errorf("test %d: Kind.String() = %v, want %v", i, got, tt.want) + } + }) } } @@ -358,7 +387,6 @@ func TestKindForGoValue(t *testing.T) { want Kind }{ {"hello", StringKind}, - {stringBuilder("hello"), StringKind}, {[]byte("hello"), BytesKind}, {int8(1), Int8Kind}, {uint8(1), Uint8Kind}, @@ -375,8 +403,10 @@ func TestKindForGoValue(t *testing.T) { {map[string]interface{}{"a": 1}, JSONKind}, } for i, tt := range tests { - if got := KindForGoValue(tt.value); got != tt.want { - t.Errorf("test %d: KindForGoValue(%v) = %v, want %v", i, tt.value, got, tt.want) - } + t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { + if got := KindForGoValue(tt.value); got != tt.want { + t.Errorf("test %d: KindForGoValue(%v) = %v, want %v", i, tt.value, got, tt.want) + } + }) } } From 29d5a2955745b17cb11e6774258ae4f3c6a89745 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 12:21:40 -0400 Subject: [PATCH 28/63] WIP updates --- indexer/base/enum.go | 13 ++++- indexer/base/field.go | 105 +++++----------------------------- indexer/base/kind.go | 6 +- indexer/base/kind_test.go | 21 +++++++ indexer/base/module_schema.go | 103 ++++++++++++++++++++++++++++++++- indexer/base/object_type.go | 3 - 6 files changed, 150 insertions(+), 101 deletions(-) diff --git a/indexer/base/enum.go b/indexer/base/enum.go index b1275d8a7a95..d9cd36428939 100644 --- a/indexer/base/enum.go +++ b/indexer/base/enum.go @@ -1,10 +1,19 @@ package indexerbase +import "fmt" + // EnumDefinition represents the definition of an enum type. type EnumDefinition struct { - // Name is the name of the enum type. - Name string // Values is a list of distinct values that are part of the enum type. Values []string } + +func (e EnumDefinition) ValidateValue(value string) error { + for _, v := range e.Values { + if v == value { + return nil + } + } + return fmt.Errorf("value %q is not a valid enum value for %s", value, e.Name) +} diff --git a/indexer/base/field.go b/indexer/base/field.go index 4a237abff3f0..7ab2804ee531 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -16,8 +16,9 @@ type Field struct { // AddressPrefix is the address prefix of the field's kind, currently only used for Bech32AddressKind. AddressPrefix string - // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. - EnumDefinition EnumDefinition + // EnumType must refer to an enum definition in the module schema and must only be set when + // the field's kind is EnumKind. + EnumType string } // Validate validates the field. @@ -39,13 +40,13 @@ func (c Field) Validate() error { return fmt.Errorf("address prefix is only valid for field %q with type Bech32AddressKind", c.Name) } - // enum definition only valid with EnumKind + // enum type only valid with EnumKind if c.Kind == EnumKind { - if err := c.EnumDefinition.Validate(); err != nil { - return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err) + if c.EnumType == "" { + return fmt.Errorf("missing EnumType for field %q", c.Name) } - } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { - return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name) + } else if c.Kind != EnumKind && c.EnumType != "" { + return fmt.Errorf("EnumType is only valid for field with type EnumKind, found it on %s", c.Name) } return nil @@ -53,37 +54,24 @@ func (c Field) Validate() error { // Validate validates the enum definition. func (e EnumDefinition) Validate() error { - if e.Name == "" { - return fmt.Errorf("enum definition name cannot be empty") - } if len(e.Values) == 0 { return fmt.Errorf("enum definition values cannot be empty") } seen := make(map[string]bool, len(e.Values)) for i, v := range e.Values { if v == "" { - return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name) + return fmt.Errorf("enum definition value at index %d cannot be empty", i) } if seen[v] { - return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name) + return fmt.Errorf("duplicate enum definition value %q", v) } seen[v] = true } return nil } -func (e EnumDefinition) ValidateValue(value string) error { - for _, v := range e.Values { - if v == value { - return nil - } - } - return fmt.Errorf("value %q is not a valid enum value for %s", value, e.Name) -} - -// ValidateValue validates that the value conforms to the field's kind and nullability. -// It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind -// values are valid for their respective types behind conforming to the correct go type. +// ValidateValue validates that the value corresponds to the field's kind and nullability, but +// cannot check for enum value correctness. func (c Field) ValidateValue(value interface{}) error { if value == nil { if !c.Nullable { @@ -91,73 +79,6 @@ func (c Field) ValidateValue(value interface{}) error { } return nil } - err := c.Kind.ValidateValueType(value) - if err != nil { - return fmt.Errorf("invalid value for field %q: %w", c.Name, err) - } - - if c.Kind == EnumKind { - return c.EnumDefinition.ValidateValue(value.(string)) - } - - return nil -} - -// ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. -// See EntityUpdate.Key for documentation on the requirements of such values. -func ValidateKey(fields []Field, value interface{}) error { - if len(fields) == 0 { - return nil - } - if len(fields) == 1 { - return fields[0].ValidateValue(value) - } - - values, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected slice of values for key fields, got %T", value) - } - - if len(fields) != len(values) { - return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]interface{}))) - } - for i, field := range fields { - if err := field.ValidateValue(values[i]); err != nil { - return fmt.Errorf("invalid value for key field %q: %w", field.Name, err) - } - } - return nil -} - -// ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate. -// See EntityUpdate.Value for documentation on the requirements of such values. -func ValidateValue(fields []Field, value interface{}) error { - valueUpdates, ok := value.(ValueUpdates) - if ok { - fieldMap := map[string]Field{} - for _, field := range fields { - fieldMap[field.Name] = field - } - var errs []error - err := valueUpdates.Iterate(func(fieldName string, value interface{}) bool { - field, ok := fieldMap[fieldName] - if !ok { - errs = append(errs, fmt.Errorf("unknown field %q in value updates", fieldName)) - } - if err := field.ValidateValue(value); err != nil { - errs = append(errs, fmt.Errorf("invalid value for field %q: %w", fieldName, err)) - } - return true - }) - if err != nil { - return err - } - if len(errs) > 0 { - return fmt.Errorf("validation errors: %v", errs) - } - return nil - } else { - return ValidateKey(fields, value) - } + return c.Kind.ValidateValue(value) } diff --git a/indexer/base/kind.go b/indexer/base/kind.go index c5abf2b75a0b..8c60b70e1760 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -78,8 +78,8 @@ const ( Bech32AddressKind // EnumKind is an enum type and values of this type must be of the go type string. - // Fields of this type are expected to set the EnumDefinition field in the field definition to the enum - // definition. + // Fields of this type are expected to set the EnumType field to the name of a valid + // enum type within the module schema. EnumKind // JSONKind is a JSON type and values of this type should be of go type json.RawMessage and represent @@ -260,7 +260,7 @@ func (t Kind) ValidateValueType(value interface{}) error { return nil } -// ValidateValue returns an error if the value does not conform to the expected go type and format. +// ValidateObjectValue returns an error if the value does not conform to the expected go type and format. // It is more thorough, but slower, than Kind.ValidateValueType and validates that Integer, Decimal and JSON // values are formatted correctly. It cannot validate enum values because Kind's do not have enum schemas. func (t Kind) ValidateValue(value interface{}) error { diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go index 80121f195892..8dce6b6d0eae 100644 --- a/indexer/base/kind_test.go +++ b/indexer/base/kind_test.go @@ -261,6 +261,27 @@ func TestKind_ValidateValueType(t *testing.T) { value: json.RawMessage("{}"), valid: true, }, + // nils get rejected + { + kind: StringKind, + value: nil, + valid: false, + }, + { + kind: BytesKind, + value: nil, + valid: false, + }, + { + kind: Int32Kind, + value: nil, + valid: false, + }, + { + kind: JSONKind, + value: nil, + valid: false, + }, } for i, tt := range tests { diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go index a93300bfd9a8..d137d849fcf7 100644 --- a/indexer/base/module_schema.go +++ b/indexer/base/module_schema.go @@ -1,7 +1,108 @@ package indexerbase +import "fmt" + // ModuleSchema represents the logical schema of a module for purposes of indexing and querying. type ModuleSchema struct { // ObjectTypes describe the types of objects that are part of the module's schema. - ObjectTypes []ObjectType + ObjectTypes map[string]ObjectType + + // EnumTypes describe the enum types that are part of the module's schema. + EnumTypes map[string]EnumDefinition +} + +func (m ModuleSchema) ValidateObjectUpdate(update ObjectUpdate) error { + if err := m.ValidateObjectKey(update.TypeName, update.Key); err != nil { + return err + } + + if update.Delete { + return nil + } + + return m.ValidateObjectValue(update.TypeName, update.Value) +} + +func (m ModuleSchema) ValidateObjectKey(objectType string, value interface{}) error { + return m.validateFieldsValue(m.ObjectTypes[objectType].KeyFields, value) +} + +func (m ModuleSchema) ValidateObjectValue(objectType string, value interface{}) error { + valueFields := m.ObjectTypes[objectType].ValueFields + + valueUpdates, ok := value.(ValueUpdates) + if !ok { + return m.validateFieldsValue(valueFields, value) + } + + values := map[string]interface{}{} + err := valueUpdates.Iterate(func(fieldName string, value interface{}) bool { + values[fieldName] = value + return true + }) + if err != nil { + return err + } + + for _, field := range valueFields { + v, ok := values[field.Name] + if !ok { + continue + } + + if err := m.validateFieldValue(field, v); err != nil { + return err + } + + delete(values, field.Name) + } + + if len(values) > 0 { + return fmt.Errorf("unexpected fields in ValueUpdates: %v", values) + } + + return nil +} + +func (m ModuleSchema) validateFieldValue(field Field, value interface{}) error { + if err := field.ValidateValue(value); err != nil { + return fmt.Errorf("invalid value for key field %q: %w", field.Name, err) + } + + if field.Kind == EnumKind { + enumType, ok := m.EnumTypes[field.EnumType] + if !ok { + return fmt.Errorf("unknown enum type %q for field %q", field.EnumType, field.Name) + } + if err := enumType.ValidateValue(value.(string)); err != nil { + return err + } + } + + return nil +} + +func (m ModuleSchema) validateFieldsValue(fields []Field, value interface{}) error { + if len(fields) == 0 { + return nil + } + + if len(fields) == 1 { + return m.validateFieldValue(fields[0], value) + } + + values, ok := value.([]interface{}) + if !ok { + return fmt.Errorf("expected slice of values for key fields, got %T", value) + } + + if len(fields) != len(values) { + return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]interface{}))) + } + for i, field := range fields { + if err := m.validateFieldValue(field, values[i]); err != nil { + return err + } + } + return nil } diff --git a/indexer/base/object_type.go b/indexer/base/object_type.go index 06f8adbc92a7..3f1c25039c47 100644 --- a/indexer/base/object_type.go +++ b/indexer/base/object_type.go @@ -2,9 +2,6 @@ package indexerbase // ObjectType describes an object type a module schema. type ObjectType struct { - // Name is the name of the object. - Name string - // KeyFields is a list of fields that make up the primary key of the object. // It can be empty in which case indexers should assume that this object is // a singleton and ony has one value. From 36aa92ddc93614ad786a956ca7b05638bc76be32 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 12:35:15 -0400 Subject: [PATCH 29/63] updates --- indexer/base/enum.go | 24 +++++++++ indexer/base/field.go | 51 ++++++++----------- indexer/base/kind.go | 4 +- indexer/base/module_schema.go | 94 +++-------------------------------- indexer/base/object_type.go | 68 +++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 119 deletions(-) diff --git a/indexer/base/enum.go b/indexer/base/enum.go index d9cd36428939..28c874ac55ef 100644 --- a/indexer/base/enum.go +++ b/indexer/base/enum.go @@ -4,11 +4,35 @@ import "fmt" // EnumDefinition represents the definition of an enum type. type EnumDefinition struct { + // Name is the name of the enum type. + Name string // Values is a list of distinct values that are part of the enum type. Values []string } +// Validate validates the enum definition. +func (e EnumDefinition) Validate() error { + if e.Name == "" { + return fmt.Errorf("enum definition name cannot be empty") + } + if len(e.Values) == 0 { + return fmt.Errorf("enum definition values cannot be empty") + } + seen := make(map[string]bool, len(e.Values)) + for i, v := range e.Values { + if v == "" { + return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name) + } + if seen[v] { + return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name) + } + seen[v] = true + } + return nil +} + +// ValidateValue validates that the value is a valid enum value. func (e EnumDefinition) ValidateValue(value string) error { for _, v := range e.Values { if v == value { diff --git a/indexer/base/field.go b/indexer/base/field.go index 7ab2804ee531..611283b1c32c 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -16,9 +16,11 @@ type Field struct { // AddressPrefix is the address prefix of the field's kind, currently only used for Bech32AddressKind. AddressPrefix string - // EnumType must refer to an enum definition in the module schema and must only be set when - // the field's kind is EnumKind. - EnumType string + // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. + // The same enum types can be reused in the same module schema, but they always must contain + // the same values for the same enum name. This possibly introduces some duplication of + // definitions but makes it easier to reason about correctness and validation in isolation. + EnumDefinition EnumDefinition } // Validate validates the field. @@ -40,38 +42,21 @@ func (c Field) Validate() error { return fmt.Errorf("address prefix is only valid for field %q with type Bech32AddressKind", c.Name) } - // enum type only valid with EnumKind + // enum definition only valid with EnumKind if c.Kind == EnumKind { - if c.EnumType == "" { - return fmt.Errorf("missing EnumType for field %q", c.Name) + if err := c.EnumDefinition.Validate(); err != nil { + return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err) } - } else if c.Kind != EnumKind && c.EnumType != "" { - return fmt.Errorf("EnumType is only valid for field with type EnumKind, found it on %s", c.Name) + } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { + return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name) } return nil } -// Validate validates the enum definition. -func (e EnumDefinition) Validate() error { - if len(e.Values) == 0 { - return fmt.Errorf("enum definition values cannot be empty") - } - seen := make(map[string]bool, len(e.Values)) - for i, v := range e.Values { - if v == "" { - return fmt.Errorf("enum definition value at index %d cannot be empty", i) - } - if seen[v] { - return fmt.Errorf("duplicate enum definition value %q", v) - } - seen[v] = true - } - return nil -} - -// ValidateValue validates that the value corresponds to the field's kind and nullability, but -// cannot check for enum value correctness. +// ValidateValue validates that the value conforms to the field's kind and nullability. +// Unlike Kind.ValidateValue, it also checks that the value conforms to the EnumDefinition +// if the field is an EnumKind. func (c Field) ValidateValue(value interface{}) error { if value == nil { if !c.Nullable { @@ -79,6 +64,14 @@ func (c Field) ValidateValue(value interface{}) error { } return nil } + err := c.Kind.ValidateValueType(value) + if err != nil { + return fmt.Errorf("invalid value for field %q: %w", c.Name, err) + } + + if c.Kind == EnumKind { + return c.EnumDefinition.ValidateValue(value.(string)) + } - return c.Kind.ValidateValue(value) + return nil } diff --git a/indexer/base/kind.go b/indexer/base/kind.go index 8c60b70e1760..9d09e1fa80a6 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -78,8 +78,8 @@ const ( Bech32AddressKind // EnumKind is an enum type and values of this type must be of the go type string. - // Fields of this type are expected to set the EnumType field to the name of a valid - // enum type within the module schema. + // Fields of this type are expected to set the EnumDefinition field in the field definition to the enum + // definition. EnumKind // JSONKind is a JSON type and values of this type should be of go type json.RawMessage and represent diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go index d137d849fcf7..efe0cf2a6f94 100644 --- a/indexer/base/module_schema.go +++ b/indexer/base/module_schema.go @@ -6,103 +6,21 @@ import "fmt" type ModuleSchema struct { // ObjectTypes describe the types of objects that are part of the module's schema. ObjectTypes map[string]ObjectType - - // EnumTypes describe the enum types that are part of the module's schema. - EnumTypes map[string]EnumDefinition } func (m ModuleSchema) ValidateObjectUpdate(update ObjectUpdate) error { - if err := m.ValidateObjectKey(update.TypeName, update.Key); err != nil { - return err - } - - if update.Delete { - return nil - } - - return m.ValidateObjectValue(update.TypeName, update.Value) -} - -func (m ModuleSchema) ValidateObjectKey(objectType string, value interface{}) error { - return m.validateFieldsValue(m.ObjectTypes[objectType].KeyFields, value) -} - -func (m ModuleSchema) ValidateObjectValue(objectType string, value interface{}) error { - valueFields := m.ObjectTypes[objectType].ValueFields - - valueUpdates, ok := value.(ValueUpdates) + objectType, ok := m.ObjectTypes[update.TypeName] if !ok { - return m.validateFieldsValue(valueFields, value) - } - - values := map[string]interface{}{} - err := valueUpdates.Iterate(func(fieldName string, value interface{}) bool { - values[fieldName] = value - return true - }) - if err != nil { - return err - } - - for _, field := range valueFields { - v, ok := values[field.Name] - if !ok { - continue - } - - if err := m.validateFieldValue(field, v); err != nil { - return err - } - - delete(values, field.Name) - } - - if len(values) > 0 { - return fmt.Errorf("unexpected fields in ValueUpdates: %v", values) + return fmt.Errorf("unknown object type %q", update.TypeName) } - return nil -} - -func (m ModuleSchema) validateFieldValue(field Field, value interface{}) error { - if err := field.ValidateValue(value); err != nil { - return fmt.Errorf("invalid value for key field %q: %w", field.Name, err) + if err := objectType.ValidateKey(update.Key); err != nil { + return fmt.Errorf("invalid key for object type %q: %w", update.TypeName, err) } - if field.Kind == EnumKind { - enumType, ok := m.EnumTypes[field.EnumType] - if !ok { - return fmt.Errorf("unknown enum type %q for field %q", field.EnumType, field.Name) - } - if err := enumType.ValidateValue(value.(string)); err != nil { - return err - } - } - - return nil -} - -func (m ModuleSchema) validateFieldsValue(fields []Field, value interface{}) error { - if len(fields) == 0 { + if update.Delete { return nil } - if len(fields) == 1 { - return m.validateFieldValue(fields[0], value) - } - - values, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected slice of values for key fields, got %T", value) - } - - if len(fields) != len(values) { - return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]interface{}))) - } - for i, field := range fields { - if err := m.validateFieldValue(field, values[i]); err != nil { - return err - } - } - return nil + return objectType.ValidateValue(update.Value) } diff --git a/indexer/base/object_type.go b/indexer/base/object_type.go index 3f1c25039c47..2dfb6e39aab9 100644 --- a/indexer/base/object_type.go +++ b/indexer/base/object_type.go @@ -1,5 +1,7 @@ package indexerbase +import "fmt" + // ObjectType describes an object type a module schema. type ObjectType struct { // KeyFields is a list of fields that make up the primary key of the object. @@ -18,3 +20,69 @@ type ObjectType struct { // the option of retaining such data and distinguishing from other "true" deletions. RetainDeletions bool } + +func (o ObjectType) ValidateKey(value interface{}) error { + return validateFieldsValue(o.KeyFields, value) +} + +// ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate. +// See EntityUpdate.Value for documentation on the requirements of such values. +func (o ObjectType) ValidateValue(value interface{}) error { + valueUpdates, ok := value.(ValueUpdates) + if !ok { + return validateFieldsValue(o.ValueFields, value) + } + + values := map[string]interface{}{} + err := valueUpdates.Iterate(func(fieldname string, value interface{}) bool { + values[fieldname] = value + return true + }) + if err != nil { + return err + } + + for _, field := range o.ValueFields { + v, ok := values[field.Name] + if !ok { + continue + } + + if err := field.ValidateValue(v); err != nil { + return err + } + + delete(values, field.Name) + } + + if len(values) > 0 { + return fmt.Errorf("unexpected fields in ValueUpdates: %v", values) + } + + return nil +} + +func validateFieldsValue(fields []Field, value interface{}) error { + if len(fields) == 0 { + return nil + } + + if len(fields) == 1 { + return fields[0].ValidateValue(value) + } + + values, ok := value.([]interface{}) + if !ok { + return fmt.Errorf("expected slice of values for key fields, got %T", value) + } + + if len(fields) != len(values) { + return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]interface{}))) + } + for i, field := range fields { + if err := field.ValidateValue(values[i]); err != nil { + return err + } + } + return nil +} From aa73bd2d47e307dd4149ec81b65ebe25ced9396f Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 13:19:19 -0400 Subject: [PATCH 30/63] updates --- indexer/base/module_schema.go | 28 ++++++++++++------------ indexer/base/object_type.go | 40 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go index efe0cf2a6f94..50f1c822e6db 100644 --- a/indexer/base/module_schema.go +++ b/indexer/base/module_schema.go @@ -5,22 +5,24 @@ import "fmt" // ModuleSchema represents the logical schema of a module for purposes of indexing and querying. type ModuleSchema struct { // ObjectTypes describe the types of objects that are part of the module's schema. - ObjectTypes map[string]ObjectType + ObjectTypes []ObjectType } -func (m ModuleSchema) ValidateObjectUpdate(update ObjectUpdate) error { - objectType, ok := m.ObjectTypes[update.TypeName] - if !ok { - return fmt.Errorf("unknown object type %q", update.TypeName) - } - - if err := objectType.ValidateKey(update.Key); err != nil { - return fmt.Errorf("invalid key for object type %q: %w", update.TypeName, err) +// Validate validates the module schema. +func (s ModuleSchema) Validate() error { + for _, objType := range s.ObjectTypes { + if err := objType.Validate(); err != nil { + return err + } } + return nil +} - if update.Delete { - return nil +func (s ModuleSchema) ValidateObjectUpdate(update ObjectUpdate) error { + for _, objType := range s.ObjectTypes { + if objType.Name == update.TypeName { + return objType.ValidateObjectUpdate(update) + } } - - return objectType.ValidateValue(update.Value) + return fmt.Errorf("object type %q not found in module schema", update.TypeName) } diff --git a/indexer/base/object_type.go b/indexer/base/object_type.go index 2dfb6e39aab9..0386e9d7540d 100644 --- a/indexer/base/object_type.go +++ b/indexer/base/object_type.go @@ -4,6 +4,9 @@ import "fmt" // ObjectType describes an object type a module schema. type ObjectType struct { + // Name is the name of the object. + Name string + // KeyFields is a list of fields that make up the primary key of the object. // It can be empty in which case indexers should assume that this object is // a singleton and ony has one value. @@ -21,6 +24,43 @@ type ObjectType struct { RetainDeletions bool } +func (o ObjectType) Validate() error { + if o.Name == "" { + return fmt.Errorf("object type name cannot be empty") + } + + for _, field := range o.KeyFields { + if err := field.Validate(); err != nil { + return fmt.Errorf("invalid key field %q: %w", field.Name, err) + } + } + + for _, field := range o.ValueFields { + if err := field.Validate(); err != nil { + return fmt.Errorf("invalid value field %q: %w", field.Name, err) + } + } + + return nil + +} + +func (o ObjectType) ValidateObjectUpdate(update ObjectUpdate) error { + if o.Name != update.TypeName { + return fmt.Errorf("object type name %q does not match update type name %q", o.Name, update.TypeName) + } + + if err := o.ValidateKey(update.Key); err != nil { + return fmt.Errorf("invalid key for object type %q: %w", update.TypeName, err) + } + + if update.Delete { + return nil + } + + return o.ValidateValue(update.Value) +} + func (o ObjectType) ValidateKey(value interface{}) error { return validateFieldsValue(o.KeyFields, value) } From 160c186aa20b78c5bb93a863de5144cbad880cae Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 13:22:28 -0400 Subject: [PATCH 31/63] update kind tests --- indexer/base/kind_test.go | 303 +++++++------------------------------- 1 file changed, 54 insertions(+), 249 deletions(-) diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go index 8dce6b6d0eae..5177c978ec88 100644 --- a/indexer/base/kind_test.go +++ b/indexer/base/kind_test.go @@ -7,25 +7,12 @@ import ( "time" ) -func TestKind_Validate(t *testing.T) { - validKinds := []Kind{ - StringKind, - BytesKind, - Int8Kind, - Uint8Kind, - Int16Kind, - Uint16Kind, - Int32Kind, - Uint32Kind, - Int64Kind, - Uint64Kind, - IntegerKind, - DecimalKind, - BoolKind, - EnumKind, - Bech32AddressKind, - } +var validKinds = []Kind{ + StringKind, BytesKind, Int8Kind, Uint8Kind, Int16Kind, Uint16Kind, Int32Kind, Uint32Kind, Int64Kind, + Uint64Kind, IntegerKind, DecimalKind, BoolKind, EnumKind, Bech32AddressKind, +} +func TestKind_Validate(t *testing.T) { for _, kind := range validKinds { if err := kind.Validate(); err != nil { t.Errorf("expected valid kind %s to pass validation, got: %v", kind, err) @@ -51,237 +38,48 @@ func TestKind_ValidateValueType(t *testing.T) { value interface{} valid bool }{ - { - kind: StringKind, - value: "hello", - valid: true, - }, - { - kind: StringKind, - value: []byte("hello"), - valid: false, - }, - { - kind: BytesKind, - value: []byte("hello"), - valid: true, - }, - { - kind: BytesKind, - value: "hello", - valid: false, - }, - { - kind: Int8Kind, - value: int8(1), - valid: true, - }, - { - kind: Int8Kind, - value: int16(1), - valid: false, - }, - { - kind: Uint8Kind, - value: uint8(1), - valid: true, - }, - { - kind: Uint8Kind, - value: uint16(1), - valid: false, - }, - { - kind: Int16Kind, - value: int16(1), - valid: true, - }, - { - kind: Int16Kind, - value: int32(1), - valid: false, - }, - { - kind: Uint16Kind, - value: uint16(1), - valid: true, - }, - { - kind: Uint16Kind, - value: uint32(1), - valid: false, - }, - { - kind: Int32Kind, - value: int32(1), - valid: true, - }, - { - kind: Int32Kind, - value: int64(1), - valid: false, - }, - { - kind: Uint32Kind, - value: uint32(1), - valid: true, - }, - { - kind: Uint32Kind, - value: uint64(1), - valid: false, - }, - { - kind: Int64Kind, - value: int64(1), - valid: true, - }, - { - kind: Int64Kind, - value: int32(1), - valid: false, - }, - { - kind: Uint64Kind, - value: uint64(1), - valid: true, - }, - { - kind: Uint64Kind, - value: uint32(1), - valid: false, - }, - { - kind: IntegerKind, - value: "1", - valid: true, - }, - { - kind: IntegerKind, - value: int32(1), - valid: false, - }, - { - kind: DecimalKind, - value: "1.0", - valid: true, - }, - { - kind: DecimalKind, - value: "1", - valid: true, - }, - { - kind: DecimalKind, - value: "1.1e4", - valid: true, - }, - { - kind: DecimalKind, - value: int32(1), - valid: false, - }, - { - kind: Bech32AddressKind, - value: []byte("hello"), - valid: true, - }, - { - kind: Bech32AddressKind, - value: 1, - valid: false, - }, - { - kind: BoolKind, - value: true, - valid: true, - }, - { - kind: BoolKind, - value: false, - valid: true, - }, - { - kind: BoolKind, - value: 1, - valid: false, - }, - { - kind: EnumKind, - value: "hello", - valid: true, - }, - { - kind: EnumKind, - value: 1, - valid: false, - }, - { - kind: TimeKind, - value: time.Now(), - valid: true, - }, - { - kind: TimeKind, - value: "hello", - valid: false, - }, - { - kind: DurationKind, - value: time.Second, - valid: true, - }, - { - kind: DurationKind, - value: "hello", - valid: false, - }, - { - kind: Float32Kind, - value: float32(1.0), - valid: true, - }, - { - kind: Float32Kind, - value: float64(1.0), - valid: false, - }, - { - kind: Float64Kind, - value: float64(1.0), - valid: true, - }, - { - kind: Float64Kind, - value: float32(1.0), - valid: false, - }, - { - kind: JSONKind, - value: json.RawMessage("{}"), - valid: true, - }, - // nils get rejected - { - kind: StringKind, - value: nil, - valid: false, - }, - { - kind: BytesKind, - value: nil, - valid: false, - }, - { - kind: Int32Kind, - value: nil, - valid: false, - }, - { - kind: JSONKind, - value: nil, - valid: false, - }, + {kind: StringKind, value: "hello", valid: true}, + {kind: StringKind, value: []byte("hello"), valid: false}, + {kind: BytesKind, value: []byte("hello"), valid: true}, + {kind: BytesKind, value: "hello", valid: false}, + {kind: Int8Kind, value: int8(1), valid: true}, + {kind: Int8Kind, value: int16(1), valid: false}, + {kind: Uint8Kind, value: uint8(1), valid: true}, + {kind: Uint8Kind, value: uint16(1), valid: false}, + {kind: Int16Kind, value: int16(1), valid: true}, + {kind: Int16Kind, value: int32(1), valid: false}, + {kind: Uint16Kind, value: uint16(1), valid: true}, + {kind: Uint16Kind, value: uint32(1), valid: false}, + {kind: Int32Kind, value: int32(1), valid: true}, + {kind: Int32Kind, value: int64(1), valid: false}, + {kind: Uint32Kind, value: uint32(1), valid: true}, + {kind: Uint32Kind, value: uint64(1), valid: false}, + {kind: Int64Kind, value: int64(1), valid: true}, + {kind: Int64Kind, value: int32(1), valid: false}, + {kind: Uint64Kind, value: uint64(1), valid: true}, + {kind: Uint64Kind, value: uint32(1), valid: false}, + {kind: IntegerKind, value: "1", valid: true}, + {kind: IntegerKind, value: int32(1), valid: false}, + {kind: DecimalKind, value: "1.0", valid: true}, + {kind: DecimalKind, value: "1", valid: true}, + {kind: DecimalKind, value: "1.1e4", valid: true}, + {kind: DecimalKind, value: int32(1), valid: false}, + {kind: Bech32AddressKind, value: []byte("hello"), valid: true}, + {kind: Bech32AddressKind, value: 1, valid: false}, + {kind: BoolKind, value: true, valid: true}, + {kind: BoolKind, value: false, valid: true}, + {kind: BoolKind, value: 1, valid: false}, + {kind: EnumKind, value: "hello", valid: true}, + {kind: EnumKind, value: 1, valid: false}, + {kind: TimeKind, value: time.Now(), valid: true}, + {kind: TimeKind, value: "hello", valid: false}, + {kind: DurationKind, value: time.Second, valid: true}, + {kind: DurationKind, value: "hello", valid: false}, + {kind: Float32Kind, value: float32(1.0), valid: true}, + {kind: Float32Kind, value: float64(1.0), valid: false}, + {kind: Float64Kind, value: float64(1.0), valid: true}, + {kind: Float64Kind, value: float32(1.0), valid: false}, + {kind: JSONKind, value: json.RawMessage("{}"), valid: true}, } for i, tt := range tests { @@ -295,6 +93,13 @@ func TestKind_ValidateValueType(t *testing.T) { } }) } + + // nils get rejected + for _, kind := range validKinds { + if err := kind.ValidateValueType(nil); err == nil { + t.Errorf("expected nil value to fail validation for kind %s", kind) + } + } } func TestKind_ValidateValue(t *testing.T) { From 00dceff86a3bc2f14d742f4134d0f8c2698e97ff Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 13:28:15 -0400 Subject: [PATCH 32/63] good kind test coverage --- indexer/base/kind.go | 9 ++------- indexer/base/kind_test.go | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/indexer/base/kind.go b/indexer/base/kind.go index 9d09e1fa80a6..ad81aaa6b4e2 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -298,10 +298,7 @@ var decimalRegex = regexp.MustCompile(DecimalFormat) // KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, // return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be -// represented as strings. Generally all values which do not have a more specific type will -// return JSONKind because the framework cannot decide at this point whether the value -// can or cannot be marshaled to JSON. This method should generally only be used as a fallback -// when the kind of a field is not specified more specifically. +// represented as strings. func KindForGoValue(value interface{}) Kind { switch value.(type) { case string: @@ -336,9 +333,7 @@ func KindForGoValue(value interface{}) Kind { return DurationKind case json.RawMessage: return JSONKind - case fmt.Stringer: - return StringKind default: - return JSONKind + return InvalidKind } } diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go index 5177c978ec88..2dc44e2b0603 100644 --- a/indexer/base/kind_test.go +++ b/indexer/base/kind_test.go @@ -80,6 +80,8 @@ func TestKind_ValidateValueType(t *testing.T) { {kind: Float64Kind, value: float64(1.0), valid: true}, {kind: Float64Kind, value: float32(1.0), valid: false}, {kind: JSONKind, value: json.RawMessage("{}"), valid: true}, + {kind: JSONKind, value: "hello", valid: false}, + {kind: InvalidKind, value: "hello", valid: false}, } for i, tt := range tests { @@ -108,9 +110,11 @@ func TestKind_ValidateValue(t *testing.T) { value interface{} valid bool }{ - // check a few basic cases + // check a few basic cases that should get caught be ValidateValueType {StringKind, "hello", true}, {Int64Kind, int64(1), true}, + {Int32Kind, "abc", false}, + {BytesKind, nil, false}, // check integer, decimal and json more thoroughly {IntegerKind, "1", true}, {IntegerKind, "0", true}, @@ -222,17 +226,26 @@ func TestKindForGoValue(t *testing.T) { {uint32(1), Uint32Kind}, {int64(1), Int64Kind}, {uint64(1), Uint64Kind}, + {float32(1.0), Float32Kind}, + {float64(1.0), Float64Kind}, {true, BoolKind}, {time.Now(), TimeKind}, {time.Second, DurationKind}, {json.RawMessage("{}"), JSONKind}, - {map[string]interface{}{"a": 1}, JSONKind}, + {map[string]interface{}{"a": 1}, InvalidKind}, } for i, tt := range tests { t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { if got := KindForGoValue(tt.value); got != tt.want { t.Errorf("test %d: KindForGoValue(%v) = %v, want %v", i, tt.value, got, tt.want) } + + // for valid kinds check valid value + if tt.want.Validate() == nil { + if err := tt.want.ValidateValue(tt.value); err != nil { + t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.want, err) + } + } }) } } From ccb9ca41ffd78f531b5864000939da8ee3f8a955 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 13:34:54 -0400 Subject: [PATCH 33/63] field tests and docs --- indexer/base/field.go | 4 +- indexer/base/field_test.go | 126 ++++++++++++++++++++++++++++++---- indexer/base/kind.go | 6 +- indexer/base/listener.go | 4 +- indexer/base/object_update.go | 2 +- 5 files changed, 119 insertions(+), 23 deletions(-) diff --git a/indexer/base/field.go b/indexer/base/field.go index 611283b1c32c..bb462230a286 100644 --- a/indexer/base/field.go +++ b/indexer/base/field.go @@ -32,7 +32,7 @@ func (c Field) Validate() error { // valid kind if err := c.Kind.Validate(); err != nil { - return fmt.Errorf("invalid field type for %q: %w", c.Name, err) + return fmt.Errorf("invalid field kind for %q: %w", c.Name, err) } // address prefix only valid with Bech32AddressKind @@ -47,7 +47,7 @@ func (c Field) Validate() error { if err := c.EnumDefinition.Validate(); err != nil { return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err) } - } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { + } else if c.Kind != EnumKind && (c.EnumDefinition.Name != "" || c.EnumDefinition.Values != nil) { return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name) } diff --git a/indexer/base/field_test.go b/indexer/base/field_test.go index c1d2e274dd7f..697260da77dc 100644 --- a/indexer/base/field_test.go +++ b/indexer/base/field_test.go @@ -1,12 +1,15 @@ package indexerbase -import "testing" +import ( + "strings" + "testing" +) func TestField_Validate(t *testing.T) { tests := []struct { - name string - field Field - error string + name string + field Field + errContains string }{ { name: "valid field", @@ -14,7 +17,7 @@ func TestField_Validate(t *testing.T) { Name: "field1", Kind: StringKind, }, - error: "", + errContains: "", }, { name: "empty name", @@ -22,15 +25,15 @@ func TestField_Validate(t *testing.T) { Name: "", Kind: StringKind, }, - error: "field name cannot be empty", + errContains: "field name cannot be empty", }, { name: "invalid kind", field: Field{ Name: "field1", - Kind: Kind(-1), + Kind: InvalidKind, }, - error: "invalid field type for \"field1\": invalid type: -1", + errContains: "invalid field kind", }, { name: "missing address prefix", @@ -38,7 +41,7 @@ func TestField_Validate(t *testing.T) { Name: "field1", Kind: Bech32AddressKind, }, - error: "missing address prefix for field \"field1\"", + errContains: "missing address prefix", }, { name: "address prefix with non-Bech32AddressKind", @@ -47,7 +50,7 @@ func TestField_Validate(t *testing.T) { Kind: StringKind, AddressPrefix: "prefix", }, - error: "address prefix is only valid for field \"field1\" with type Bech32AddressKind", + errContains: "address prefix is only valid for field \"field1\" with type Bech32AddressKind", }, { name: "invalid enum definition", @@ -55,7 +58,7 @@ func TestField_Validate(t *testing.T) { Name: "field1", Kind: EnumKind, }, - error: "invalid enum definition for field \"field1\": enum definition name cannot be empty", + errContains: "invalid enum definition for field \"field1\": enum definition name cannot be empty", }, { name: "enum definition with non-EnumKind", @@ -64,22 +67,115 @@ func TestField_Validate(t *testing.T) { Kind: StringKind, EnumDefinition: EnumDefinition{Name: "enum"}, }, - error: "enum definition is only valid for field \"field1\" with type EnumKind", + errContains: "enum definition is only valid for field \"field1\" with type EnumKind", + }, + { + name: "valid enum", + field: Field{ + Name: "field1", + Kind: EnumKind, + EnumDefinition: EnumDefinition{Name: "enum", Values: []string{"a", "b"}}, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := tt.field.Validate() - if tt.error == "" { + if tt.errContains == "" { + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + } else { + if err == nil { + t.Errorf("expected error, got nil") + } else if !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("expected error contains: %s, got: %v", tt.errContains, err) + } + } + }) + } +} + +func TestField_ValidateValue(t *testing.T) { + tests := []struct { + name string + field Field + value interface{} + errContains string + }{ + { + name: "valid field", + field: Field{ + Name: "field1", + Kind: StringKind, + }, + value: "value", + errContains: "", + }, + { + name: "null non-nullable field", + field: Field{ + Name: "field1", + Kind: StringKind, + Nullable: false, + }, + value: nil, + errContains: "cannot be null", + }, + { + name: "null nullable field", + field: Field{ + Name: "field1", + Kind: StringKind, + Nullable: true, + }, + value: nil, + errContains: "", + }, + { + name: "invalid value", + field: Field{ + Name: "field1", + Kind: StringKind, + }, + value: 1, + errContains: "invalid value for field \"field1\"", + }, + { + name: "valid enum", + field: Field{ + Name: "field1", + Kind: EnumKind, + EnumDefinition: EnumDefinition{Name: "enum", Values: []string{"a", "b"}}, + }, + value: "a", + errContains: "", + }, + { + name: "invalid enum", + field: Field{ + Name: "field1", + Kind: EnumKind, + EnumDefinition: EnumDefinition{Name: "enum", Values: []string{"a", "b"}}, + }, + value: "c", + errContains: "not a valid enum value", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.field.ValidateValue(tt.value) + if tt.errContains == "" { if err != nil { t.Errorf("expected no error, got: %v", err) } } else { if err == nil { t.Errorf("expected error, got nil") - } else if err.Error() != tt.error { - t.Errorf("expected error: %s, got: %v", tt.error, err) + } else if !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("expected error contains: %s, got: %v", tt.errContains, err) } } }) diff --git a/indexer/base/kind.go b/indexer/base/kind.go index ad81aaa6b4e2..0083f1e3836d 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -87,7 +87,7 @@ const ( JSONKind ) -// Validate returns an error if the kind is invalid. +// Validate returns an errContains if the kind is invalid. func (t Kind) Validate() error { if t <= InvalidKind { return fmt.Errorf("unknown type: %d", t) @@ -146,7 +146,7 @@ func (t Kind) String() string { } } -// ValidateValueType returns an error if the value does not conform to the expected go type. +// ValidateValueType returns an errContains if the value does not conform to the expected go type. // Some fields may accept nil values, however, this method does not have any notion of // nullability. This method only validates that the go type of the value is correct for the kind // and does not validate string or json formats. Kind.ValidateValue does a more thorough validation @@ -260,7 +260,7 @@ func (t Kind) ValidateValueType(value interface{}) error { return nil } -// ValidateObjectValue returns an error if the value does not conform to the expected go type and format. +// ValidateValue returns an errContains if the value does not conform to the expected go type and format. // It is more thorough, but slower, than Kind.ValidateValueType and validates that Integer, Decimal and JSON // values are formatted correctly. It cannot validate enum values because Kind's do not have enum schemas. func (t Kind) ValidateValue(value interface{}) error { diff --git a/indexer/base/listener.go b/indexer/base/listener.go index f0accb620824..2a3a2bbe005a 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -34,7 +34,7 @@ type Listener struct { OnKVPair func(moduleName string, key, value []byte, delete bool) error // Commit is called when state is committed, usually at the end of a block. Any - // indexers should commit their data when this is called and return an error if + // indexers should commit their data when this is called and return an errContains if // they are unable to commit. Commit func() error @@ -43,7 +43,7 @@ type Listener struct { // should ensure that they have performed whatever initialization steps (such as database // migrations) required to receive OnObjectUpdate events for the given module. If the // indexer's schema is incompatible with the module's on-chain schema, the listener should return - // an error. + // an errContains. InitializeModuleSchema func(module string, schema ModuleSchema) error // OnObjectUpdate is called whenever an object is updated in a module's state. This is only called diff --git a/indexer/base/object_update.go b/indexer/base/object_update.go index 05b596293080..2d4190ca9542 100644 --- a/indexer/base/object_update.go +++ b/indexer/base/object_update.go @@ -34,7 +34,7 @@ type ObjectUpdate struct { type ValueUpdates interface { // Iterate iterates over the fields and values in the object update. The function should return // true to continue iteration or false to stop iteration. Each field value should conform - // to the requirements of that field's type in the schema. Iterate returns an error if + // to the requirements of that field's type in the schema. Iterate returns an errContains if // it was unable to decode the values properly (which could be the case in lazy evaluation). Iterate(func(col string, value interface{}) bool) error } From df727e2553613ed607a0759821dd32d7d2552f64 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 13:39:25 -0400 Subject: [PATCH 34/63] enum tests --- indexer/base/enum.go | 2 +- indexer/base/enum_test.go | 106 ++++++++++++++++++++++++++++++++++++++ indexer/base/kind.go | 10 ++-- 3 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 indexer/base/enum_test.go diff --git a/indexer/base/enum.go b/indexer/base/enum.go index 28c874ac55ef..d6586b7c9824 100644 --- a/indexer/base/enum.go +++ b/indexer/base/enum.go @@ -7,7 +7,7 @@ type EnumDefinition struct { // Name is the name of the enum type. Name string - // Values is a list of distinct values that are part of the enum type. + // Values is a list of distinct, non-empty values that are part of the enum type. Values []string } diff --git a/indexer/base/enum_test.go b/indexer/base/enum_test.go new file mode 100644 index 000000000000..fddb7b90ac3d --- /dev/null +++ b/indexer/base/enum_test.go @@ -0,0 +1,106 @@ +package indexerbase + +import ( + "strings" + "testing" +) + +func TestEnumDefinition_Validate(t *testing.T) { + tests := []struct { + name string + enum EnumDefinition + errContains string + }{ + { + name: "valid enum", + enum: EnumDefinition{ + Name: "test", + Values: []string{"a", "b", "c"}, + }, + errContains: "", + }, + { + name: "empty name", + enum: EnumDefinition{ + Name: "", + Values: []string{"a", "b", "c"}, + }, + errContains: "enum definition name cannot be empty", + }, + { + name: "empty values", + enum: EnumDefinition{ + Name: "test", + Values: []string{}, + }, + errContains: "enum definition values cannot be empty", + }, + { + name: "empty value", + enum: EnumDefinition{ + Name: "test", + Values: []string{"a", "", "c"}, + }, + errContains: "enum definition value at index 1 cannot be empty for enum test", + }, + { + name: "duplicate value", + enum: EnumDefinition{ + Name: "test", + Values: []string{"a", "b", "a"}, + }, + errContains: "duplicate enum definition value \"a\" for enum test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.enum.Validate() + if tt.errContains == "" { + if err != nil { + t.Errorf("expected valid enum definition to pass validation, got: %v", err) + } + } else { + if err == nil { + t.Errorf("expected invalid enum definition to fail validation, got nil error") + } else if !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("expected error to contain %s, got: %v", tt.errContains, err) + } + } + }) + } +} + +func TestEnumDefinition_ValidateValue(t *testing.T) { + enum := EnumDefinition{ + Name: "test", + Values: []string{"a", "b", "c"}, + } + + tests := []struct { + value string + errContains string + }{ + {"a", ""}, + {"b", ""}, + {"c", ""}, + {"d", "value \"d\" is not a valid enum value for test"}, + } + + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + err := enum.ValidateValue(tt.value) + if tt.errContains == "" { + if err != nil { + t.Errorf("expected valid enum value to pass validation, got: %v", err) + } + } else { + if err == nil { + t.Errorf("expected invalid enum value to fail validation, got nil error") + } else if !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("expected error to contain %s, got: %v", tt.errContains, err) + } + } + }) + } +} diff --git a/indexer/base/kind.go b/indexer/base/kind.go index 0083f1e3836d..3f3f2f75d296 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -87,6 +87,11 @@ const ( JSONKind ) +const ( + IntegerFormat = `^-?[0-9]+$` + DecimalFormat = `^-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$` +) + // Validate returns an errContains if the kind is invalid. func (t Kind) Validate() error { if t <= InvalidKind { @@ -288,11 +293,6 @@ func (t Kind) ValidateValue(value interface{}) error { return nil } -const ( - IntegerFormat = `^-?[0-9]+$` - DecimalFormat = `^-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$` -) - var integerRegex = regexp.MustCompile(IntegerFormat) var decimalRegex = regexp.MustCompile(DecimalFormat) From 842d420d1207ee7f6ef0b81f1ea0d189417800e1 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 14:06:45 -0400 Subject: [PATCH 35/63] object type tests --- indexer/base/module_schema_test.go | 64 +++++ indexer/base/object_type.go | 6 +- indexer/base/object_type_test.go | 439 +++++++++++++++++++++++++++++ indexer/base/object_update.go | 21 ++ indexer/base/object_update_test.go | 52 ++++ 5 files changed, 581 insertions(+), 1 deletion(-) create mode 100644 indexer/base/module_schema_test.go create mode 100644 indexer/base/object_type_test.go create mode 100644 indexer/base/object_update_test.go diff --git a/indexer/base/module_schema_test.go b/indexer/base/module_schema_test.go new file mode 100644 index 000000000000..3624d6f334c3 --- /dev/null +++ b/indexer/base/module_schema_test.go @@ -0,0 +1,64 @@ +package indexerbase + +import ( + "strings" + "testing" +) + +func TestModuleSchema_Validate(t *testing.T) { + tests := []struct { + name string + moduleSchema ModuleSchema + errContains string + }{ + { + name: "valid module schema", + moduleSchema: ModuleSchema{ + ObjectTypes: []ObjectType{ + { + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + }, + }, + errContains: "", + }, + { + name: "invalid module schema", + moduleSchema: ModuleSchema{ + ObjectTypes: []ObjectType{ + { + Name: "", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + }, + }, + errContains: "object type name cannot be empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.moduleSchema.Validate() + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} diff --git a/indexer/base/object_type.go b/indexer/base/object_type.go index 0386e9d7540d..5abce90fdae0 100644 --- a/indexer/base/object_type.go +++ b/indexer/base/object_type.go @@ -41,6 +41,10 @@ func (o ObjectType) Validate() error { } } + if len(o.KeyFields) == 0 && len(o.ValueFields) == 0 { + return fmt.Errorf("object type %q has no key or value fields", o.Name) + } + return nil } @@ -96,7 +100,7 @@ func (o ObjectType) ValidateValue(value interface{}) error { } if len(values) > 0 { - return fmt.Errorf("unexpected fields in ValueUpdates: %v", values) + return fmt.Errorf("unexpected values in ValueUpdates: %v", values) } return nil diff --git a/indexer/base/object_type_test.go b/indexer/base/object_type_test.go new file mode 100644 index 000000000000..d221737bd769 --- /dev/null +++ b/indexer/base/object_type_test.go @@ -0,0 +1,439 @@ +package indexerbase + +import ( + "strings" + "testing" +) + +func TestObjectType_Validate(t *testing.T) { + tests := []struct { + name string + objectType ObjectType + errContains string + }{ + { + name: "valid object type", + objectType: ObjectType{ + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + errContains: "", + }, + { + name: "empty object type name", + objectType: ObjectType{ + Name: "", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + errContains: "object type name cannot be empty", + }, + { + name: "invalid key field", + objectType: ObjectType{ + Name: "object1", + KeyFields: []Field{ + { + Name: "", + Kind: StringKind, + }, + }, + }, + errContains: "field name cannot be empty", + }, + { + name: "invalid value field", + objectType: ObjectType{ + Name: "object1", + ValueFields: []Field{ + { + Kind: StringKind, + }, + }, + }, + errContains: "field name cannot be empty", + }, + { + name: "no fields", + objectType: ObjectType{ + Name: "object1", + }, + errContains: "has no key or value fields", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.objectType.Validate() + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} + +func TestObjectType_ValidateKey(t *testing.T) { + tests := []struct { + name string + objectType ObjectType + key interface{} + errContains string + }{ + { + name: "no key fields", + objectType: ObjectType{ + Name: "object1", + }, + key: nil, + }, + { + name: "single key field, valid", + objectType: ObjectType{ + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + key: "hello", + errContains: "", + }, + { + name: "single key field, invalid", + objectType: ObjectType{ + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + key: []interface{}{"value"}, + errContains: "invalid value", + }, + { + name: "multiple key fields, valid", + objectType: ObjectType{ + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + key: []interface{}{"hello", int32(42)}, + }, + { + name: "multiple key fields, not a slice", + objectType: ObjectType{ + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + key: map[string]interface{}{"field1": "hello", "field2": "42"}, + errContains: "expected slice of values", + }, + { + name: "multiple key fields, wrong number of values", + objectType: ObjectType{ + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + key: []interface{}{"hello"}, + errContains: "expected 2 key fields", + }, + { + name: "multiple key fields, invalid value", + objectType: ObjectType{ + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + key: []interface{}{"hello", "abc"}, + errContains: "invalid value", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.objectType.ValidateKey(tt.key) + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} + +func TestObjectType_ValidateValue(t *testing.T) { + tests := []struct { + name string + objectType ObjectType + value interface{} + errContains string + }{ + { + name: "no value fields", + objectType: ObjectType{ + Name: "object1", + }, + value: nil, + }, + { + name: "single value field, valid", + objectType: ObjectType{ + ValueFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + value: "hello", + errContains: "", + }, + { + name: "value updates, empty", + objectType: ObjectType{ + ValueFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + value: MapValueUpdates(map[string]interface{}{}), + }, + { + name: "value updates, 1 field valid", + objectType: ObjectType{ + ValueFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + value: MapValueUpdates(map[string]interface{}{ + "field1": "hello", + }), + }, + { + name: "value updates, 2 fields, 1 invalid", + objectType: ObjectType{ + ValueFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + value: MapValueUpdates(map[string]interface{}{ + "field1": "hello", + "field2": "abc", + }), + errContains: "expected int32", + }, + { + name: "value updates, extra value", + objectType: ObjectType{ + ValueFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, + }, + value: MapValueUpdates(map[string]interface{}{ + "field1": "hello", + "field2": int32(42), + "field3": "extra", + }), + errContains: "unexpected values", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.objectType.ValidateValue(tt.value) + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} + +func TestObjectType_ValidateObjectUpdate(t *testing.T) { + tests := []struct { + name string + objectType ObjectType + object ObjectUpdate + errContains string + }{ + { + name: "wrong name", + objectType: ObjectType{ + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + object: ObjectUpdate{ + TypeName: "object2", + Key: "hello", + }, + errContains: "does not match update type name", + }, + { + name: "invalid value", + objectType: ObjectType{ + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + object: ObjectUpdate{ + TypeName: "object1", + Key: 123, + }, + errContains: "invalid value", + }, + { + name: "valid update", + objectType: ObjectType{ + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: Int32Kind, + }, + }, + ValueFields: []Field{ + { + Name: "field2", + Kind: StringKind, + }, + }, + }, + object: ObjectUpdate{ + TypeName: "object1", + Key: int32(123), + Value: "hello", + }, + }, + { + name: "valid deletion", + objectType: ObjectType{ + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: Int32Kind, + }, + }, + ValueFields: []Field{ + { + Name: "field2", + Kind: StringKind, + }, + }, + }, + object: ObjectUpdate{ + TypeName: "object1", + Key: int32(123), + Value: "ignored!", + Delete: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.objectType.ValidateObjectUpdate(tt.object) + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} diff --git a/indexer/base/object_update.go b/indexer/base/object_update.go index 2d4190ca9542..38871e78baf3 100644 --- a/indexer/base/object_update.go +++ b/indexer/base/object_update.go @@ -1,5 +1,7 @@ package indexerbase +import "sort" + // ObjectUpdate represents an update operation on an object in a module's state. type ObjectUpdate struct { // TypeName is the name of the object type in the module's schema. @@ -38,3 +40,22 @@ type ValueUpdates interface { // it was unable to decode the values properly (which could be the case in lazy evaluation). Iterate(func(col string, value interface{}) bool) error } + +// MapValueUpdates is a map-based implementation of ValueUpdates which always iterates +// over keys in sorted order. +type MapValueUpdates map[string]interface{} + +// Iterate implements the ValueUpdates interface. +func (m MapValueUpdates) Iterate(fn func(col string, value interface{}) bool) error { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if !fn(k, m[k]) { + return nil + } + } + return nil +} diff --git a/indexer/base/object_update_test.go b/indexer/base/object_update_test.go new file mode 100644 index 000000000000..bfd6fbb984a0 --- /dev/null +++ b/indexer/base/object_update_test.go @@ -0,0 +1,52 @@ +package indexerbase + +import "testing" + +func TestMapValueUpdates_Iterate(t *testing.T) { + updates := MapValueUpdates(map[string]interface{}{ + "a": "abc", + "b": 123, + }) + + got := map[string]interface{}{} + err := updates.Iterate(func(fieldname string, value interface{}) bool { + got[fieldname] = value + return true + }) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if len(got) != 2 { + t.Errorf("expected 2 updates, got: %v", got) + } + + if got["a"] != "abc" { + t.Errorf("expected a=abc, got: %v", got) + } + + if got["b"] != 123 { + t.Errorf("expected b=123, got: %v", got) + } + + got = map[string]interface{}{} + err = updates.Iterate(func(fieldname string, value interface{}) bool { + if len(got) == 1 { + return false + } + got[fieldname] = value + return true + }) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if len(got) != 1 { + t.Errorf("expected 1 updates, got: %v", got) + } + + // should have gotten the first field in order + if got["a"] != "abc" { + t.Errorf("expected a=abc, got: %v", got) + } +} From 4b18658cf2fb2663cb435020c70acaa71b51e8e9 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 14:07:32 -0400 Subject: [PATCH 36/63] object type docs --- indexer/base/object_type.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/indexer/base/object_type.go b/indexer/base/object_type.go index 5abce90fdae0..342e7ef2bd93 100644 --- a/indexer/base/object_type.go +++ b/indexer/base/object_type.go @@ -24,6 +24,7 @@ type ObjectType struct { RetainDeletions bool } +// Validate validates the object type. func (o ObjectType) Validate() error { if o.Name == "" { return fmt.Errorf("object type name cannot be empty") @@ -49,6 +50,7 @@ func (o ObjectType) Validate() error { } +// ValidateObjectUpdate validates that the update conforms to the object type. func (o ObjectType) ValidateObjectUpdate(update ObjectUpdate) error { if o.Name != update.TypeName { return fmt.Errorf("object type name %q does not match update type name %q", o.Name, update.TypeName) @@ -65,6 +67,8 @@ func (o ObjectType) ValidateObjectUpdate(update ObjectUpdate) error { return o.ValidateValue(update.Value) } +// ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. +// See EntityUpdate.Key for documentation on the requirements of such keys. func (o ObjectType) ValidateKey(value interface{}) error { return validateFieldsValue(o.KeyFields, value) } From 40b5d25f54105a8f38477944476f0dfcb9ff3e66 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 14:08:55 -0400 Subject: [PATCH 37/63] object type docs --- indexer/base/module_schema.go | 1 + indexer/base/module_schema_test.go | 67 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go index 50f1c822e6db..d47c62c7d462 100644 --- a/indexer/base/module_schema.go +++ b/indexer/base/module_schema.go @@ -18,6 +18,7 @@ func (s ModuleSchema) Validate() error { return nil } +// ValidateObjectUpdate validates that the update conforms to the module schema. func (s ModuleSchema) ValidateObjectUpdate(update ObjectUpdate) error { for _, objType := range s.ObjectTypes { if objType.Name == update.TypeName { diff --git a/indexer/base/module_schema_test.go b/indexer/base/module_schema_test.go index 3624d6f334c3..a27d12bb7e73 100644 --- a/indexer/base/module_schema_test.go +++ b/indexer/base/module_schema_test.go @@ -62,3 +62,70 @@ func TestModuleSchema_Validate(t *testing.T) { }) } } + +func TestModuleSchema_ValidateObjectUpdate(t *testing.T) { + tests := []struct { + name string + moduleSchema ModuleSchema + objectUpdate ObjectUpdate + errContains string + }{ + { + name: "valid object update", + moduleSchema: ModuleSchema{ + ObjectTypes: []ObjectType{ + { + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + }, + }, + objectUpdate: ObjectUpdate{ + TypeName: "object1", + Key: "abc", + }, + errContains: "", + }, + { + name: "object type not found", + moduleSchema: ModuleSchema{ + ObjectTypes: []ObjectType{ + { + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + }, + }, + }, + objectUpdate: ObjectUpdate{ + TypeName: "object2", + Key: "abc", + }, + errContains: "object type \"object2\" not found in module schema", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.moduleSchema.ValidateObjectUpdate(tt.objectUpdate) + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} From 6ed4d2bc31779f6813fc22d4d45dffcc6ada7b29 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 14:11:38 -0400 Subject: [PATCH 38/63] revert comment changes --- indexer/base/listener.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexer/base/listener.go b/indexer/base/listener.go index 2a3a2bbe005a..f0accb620824 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -34,7 +34,7 @@ type Listener struct { OnKVPair func(moduleName string, key, value []byte, delete bool) error // Commit is called when state is committed, usually at the end of a block. Any - // indexers should commit their data when this is called and return an errContains if + // indexers should commit their data when this is called and return an error if // they are unable to commit. Commit func() error @@ -43,7 +43,7 @@ type Listener struct { // should ensure that they have performed whatever initialization steps (such as database // migrations) required to receive OnObjectUpdate events for the given module. If the // indexer's schema is incompatible with the module's on-chain schema, the listener should return - // an errContains. + // an error. InitializeModuleSchema func(module string, schema ModuleSchema) error // OnObjectUpdate is called whenever an object is updated in a module's state. This is only called From 3ac7b0a75516aea7be90044c0132bc7757477d16 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 14:12:26 -0400 Subject: [PATCH 39/63] revert comment changes --- indexer/base/object_update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/base/object_update.go b/indexer/base/object_update.go index 38871e78baf3..73e9c5ce4e37 100644 --- a/indexer/base/object_update.go +++ b/indexer/base/object_update.go @@ -36,7 +36,7 @@ type ObjectUpdate struct { type ValueUpdates interface { // Iterate iterates over the fields and values in the object update. The function should return // true to continue iteration or false to stop iteration. Each field value should conform - // to the requirements of that field's type in the schema. Iterate returns an errContains if + // to the requirements of that field's type in the schema. Iterate returns an error if // it was unable to decode the values properly (which could be the case in lazy evaluation). Iterate(func(col string, value interface{}) bool) error } From 8f3391be264fb62bb3e1966c691ed2feb034686e Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 14:18:06 -0400 Subject: [PATCH 40/63] simplify tests --- indexer/base/object_type_test.go | 289 ++++++++++--------------------- 1 file changed, 92 insertions(+), 197 deletions(-) diff --git a/indexer/base/object_type_test.go b/indexer/base/object_type_test.go index d221737bd769..89aab6dbf6c0 100644 --- a/indexer/base/object_type_test.go +++ b/indexer/base/object_type_test.go @@ -5,6 +5,59 @@ import ( "testing" ) +var object1Type = ObjectType{ + Name: "object1", + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, +} + +var object2Type = ObjectType{ + KeyFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, +} + +var object3Type = ObjectType{ + Name: "object3", + ValueFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + { + Name: "field2", + Kind: Int32Kind, + }, + }, +} + +var object4Type = ObjectType{ + Name: "object4", + KeyFields: []Field{ + { + Name: "field1", + Kind: Int32Kind, + }, + }, + ValueFields: []Field{ + { + Name: "field2", + Kind: StringKind, + }, + }, +} + func TestObjectType_Validate(t *testing.T) { tests := []struct { name string @@ -12,16 +65,8 @@ func TestObjectType_Validate(t *testing.T) { errContains string }{ { - name: "valid object type", - objectType: ObjectType{ - Name: "object1", - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - }, - }, + name: "valid object type", + objectType: object1Type, errContains: "", }, { @@ -63,10 +108,8 @@ func TestObjectType_Validate(t *testing.T) { errContains: "field name cannot be empty", }, { - name: "no fields", - objectType: ObjectType{ - Name: "object1", - }, + name: "no fields", + objectType: ObjectType{Name: "object0"}, errContains: "has no key or value fields", }, } @@ -102,95 +145,37 @@ func TestObjectType_ValidateKey(t *testing.T) { key: nil, }, { - name: "single key field, valid", - objectType: ObjectType{ - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - }, - }, + name: "single key field, valid", + objectType: object1Type, key: "hello", errContains: "", }, { - name: "single key field, invalid", - objectType: ObjectType{ - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - }, - }, + name: "single key field, invalid", + objectType: object1Type, key: []interface{}{"value"}, errContains: "invalid value", }, { - name: "multiple key fields, valid", - objectType: ObjectType{ - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, - key: []interface{}{"hello", int32(42)}, + name: "multiple key fields, valid", + objectType: object2Type, + key: []interface{}{"hello", int32(42)}, }, { - name: "multiple key fields, not a slice", - objectType: ObjectType{ - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, + name: "multiple key fields, not a slice", + objectType: object2Type, key: map[string]interface{}{"field1": "hello", "field2": "42"}, errContains: "expected slice of values", }, { - name: "multiple key fields, wrong number of values", - objectType: ObjectType{ - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, + name: "multiple key fields, wrong number of values", + objectType: object2Type, key: []interface{}{"hello"}, errContains: "expected 2 key fields", }, { - name: "multiple key fields, invalid value", - objectType: ObjectType{ - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, + name: "multiple key fields, invalid value", + objectType: object2Type, key: []interface{}{"hello", "abc"}, errContains: "invalid value", }, @@ -220,11 +205,9 @@ func TestObjectType_ValidateValue(t *testing.T) { errContains string }{ { - name: "no value fields", - objectType: ObjectType{ - Name: "object1", - }, - value: nil, + name: "no value fields", + objectType: ObjectType{Name: "object0"}, + value: nil, }, { name: "single value field, valid", @@ -240,53 +223,20 @@ func TestObjectType_ValidateValue(t *testing.T) { errContains: "", }, { - name: "value updates, empty", - objectType: ObjectType{ - ValueFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, - value: MapValueUpdates(map[string]interface{}{}), + name: "value updates, empty", + objectType: object3Type, + value: MapValueUpdates(map[string]interface{}{}), }, { - name: "value updates, 1 field valid", - objectType: ObjectType{ - ValueFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, + name: "value updates, 1 field valid", + objectType: object3Type, value: MapValueUpdates(map[string]interface{}{ "field1": "hello", }), }, { - name: "value updates, 2 fields, 1 invalid", - objectType: ObjectType{ - ValueFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, + name: "value updates, 2 fields, 1 invalid", + objectType: object3Type, value: MapValueUpdates(map[string]interface{}{ "field1": "hello", "field2": "abc", @@ -294,19 +244,8 @@ func TestObjectType_ValidateValue(t *testing.T) { errContains: "expected int32", }, { - name: "value updates, extra value", - objectType: ObjectType{ - ValueFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - { - Name: "field2", - Kind: Int32Kind, - }, - }, - }, + name: "value updates, extra value", + objectType: object3Type, value: MapValueUpdates(map[string]interface{}{ "field1": "hello", "field2": int32(42), @@ -340,16 +279,8 @@ func TestObjectType_ValidateObjectUpdate(t *testing.T) { errContains string }{ { - name: "wrong name", - objectType: ObjectType{ - Name: "object1", - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - }, - }, + name: "wrong name", + objectType: object1Type, object: ObjectUpdate{ TypeName: "object2", Key: "hello", @@ -357,16 +288,8 @@ func TestObjectType_ValidateObjectUpdate(t *testing.T) { errContains: "does not match update type name", }, { - name: "invalid value", - objectType: ObjectType{ - Name: "object1", - KeyFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - }, - }, + name: "invalid value", + objectType: object1Type, object: ObjectUpdate{ TypeName: "object1", Key: 123, @@ -374,47 +297,19 @@ func TestObjectType_ValidateObjectUpdate(t *testing.T) { errContains: "invalid value", }, { - name: "valid update", - objectType: ObjectType{ - Name: "object1", - KeyFields: []Field{ - { - Name: "field1", - Kind: Int32Kind, - }, - }, - ValueFields: []Field{ - { - Name: "field2", - Kind: StringKind, - }, - }, - }, + name: "valid update", + objectType: object4Type, object: ObjectUpdate{ - TypeName: "object1", + TypeName: "object4", Key: int32(123), Value: "hello", }, }, { - name: "valid deletion", - objectType: ObjectType{ - Name: "object1", - KeyFields: []Field{ - { - Name: "field1", - Kind: Int32Kind, - }, - }, - ValueFields: []Field{ - { - Name: "field2", - Kind: StringKind, - }, - }, - }, + name: "valid deletion", + objectType: object4Type, object: ObjectUpdate{ - TypeName: "object1", + TypeName: "object4", Key: int32(123), Value: "ignored!", Delete: true, From 5edf810c8225d74ce3a0481700ca6950b75c8fc9 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 17:13:44 -0400 Subject: [PATCH 41/63] updates --- collections/codec/codec.go | 15 +++++++++++++++ collections/collections.go | 9 ++++++--- collections/indexing.go | 25 +++++++++++++------------ collections/map.go | 7 ++++--- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/collections/codec/codec.go b/collections/codec/codec.go index 2988c9f52425..6f58c31e3469 100644 --- a/collections/codec/codec.go +++ b/collections/codec/codec.go @@ -3,6 +3,8 @@ package codec import ( "errors" "fmt" + + indexerbase "cosmossdk.io/indexer/base" ) var ErrEncoding = errors.New("collections: encoding error") @@ -74,6 +76,19 @@ type ValueCodec[T any] interface { ValueType() string } +// IndexableCodec is an interface that all codec's should implement in order to properly support indexing. +// It is not required by KeyCodec or ValueCodec in order to preserve backwards compatibility, but +// a future version of collections may make it required and all codec's should aim to implement it. +// If it is not implemented, fallback defaults will be used for indexing that may be sub-optimal. +type IndexableCodec interface { + + // SchemaFields returns the logical schema fields of the codec's type. + SchemaFields() []indexerbase.Field + + // DecodeIndexable decodes the provided bytes into an indexable value that conforms to the codec's schema fields. + DecodeIndexable([]byte) (any, error) +} + // NewUntypedValueCodec returns an UntypedValueCodec for the provided ValueCodec. func NewUntypedValueCodec[V any](v ValueCodec[V]) UntypedValueCodec { typeName := fmt.Sprintf("%T", *new(V)) diff --git a/collections/collections.go b/collections/collections.go index cb5870d07afa..f56ae1bbae3d 100644 --- a/collections/collections.go +++ b/collections/collections.go @@ -7,6 +7,7 @@ import ( "math" "cosmossdk.io/collections/codec" + indexerbase "cosmossdk.io/indexer/base" ) var ( @@ -91,11 +92,11 @@ type Collection interface { genesisHandler - getTableSchema() indexerbase.Table + objectType() indexerbase.ObjectType - decodeKVPair(key, value []byte, delete bool) (indexerbase.EntityUpdate, bool, error) + decodeKVPair(key, value []byte, delete bool) (indexerbase.ObjectUpdate, bool, error) - isIndex() bool + isSecondaryIndex() bool } // Prefix defines a segregation bytes namespace for specific collections objects. @@ -163,3 +164,5 @@ func (c collectionImpl[K, V]) exportGenesis(ctx context.Context, w io.Writer) er } func (c collectionImpl[K, V]) defaultGenesis(w io.Writer) error { return c.m.defaultGenesis(w) } + +func (c collectionImpl[K, V]) isSecondaryIndex() bool { return c.m.isSecondaryIndex } diff --git a/collections/indexing.go b/collections/indexing.go index 12037a6cf130..87a0ebfea4b5 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -7,6 +7,7 @@ import ( "github.com/tidwall/btree" + "cosmossdk.io/collections/codec" indexerbase "cosmossdk.io/indexer/base" ) @@ -22,11 +23,11 @@ func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, var objectTypes []indexerbase.ObjectType for _, collName := range s.collectionsOrdered { coll := s.collectionsByName[collName] - if coll.isIndex() { + if coll.isSecondaryIndex() { continue } - schema := coll.getTableSchema() + schema := coll.objectType() objectTypes = append(objectTypes, schema) decoder.lookup.Set(string(coll.GetPrefix()), &collDecoder{Collection: coll}) } @@ -91,7 +92,7 @@ func (c collectionImpl[K, V]) getTableSchema() indexerbase.ObjectType { var keyFields []indexerbase.Field var valueFields []indexerbase.Field - if hasSchema, ok := c.m.kc.(IndexableCodec); ok { + if hasSchema, ok := c.m.kc.(codec.IndexableCodec); ok { keyFields = hasSchema.SchemaFields() } else { var k K @@ -99,7 +100,7 @@ func (c collectionImpl[K, V]) getTableSchema() indexerbase.ObjectType { } ensureNames(c.m.kc, "key", keyFields) - if hasSchema, ok := c.m.vc.(IndexableCodec); ok { + if hasSchema, ok := c.m.vc.(codec.IndexableCodec); ok { valueFields = hasSchema.SchemaFields() } else { var v V @@ -115,7 +116,7 @@ func (c collectionImpl[K, V]) getTableSchema() indexerbase.ObjectType { } func extractFields(x any) ([]indexerbase.Field, func(any) any) { - if hasSchema, ok := x.(IndexableCodec); ok { + if hasSchema, ok := x.(codec.IndexableCodec); ok { return hasSchema.SchemaFields(), nil } @@ -151,12 +152,17 @@ func ensureNames(x any, defaultName string, cols []indexerbase.Field) { } } +func (c collectionImpl[K, V]) objectType() indexerbase.ObjectType { + //TODO implement me + panic("implement me") +} + func (c collectionImpl[K, V]) decodeKVPair(key, value []byte, delete bool) (indexerbase.ObjectUpdate, bool, error) { // strip prefix key = key[len(c.GetPrefix()):] var k any var err error - if decodeAny, ok := c.m.kc.(IndexableCodec); ok { + if decodeAny, ok := c.m.kc.(codec.IndexableCodec); ok { k, err = decodeAny.DecodeIndexable(key) } else { _, k, err = c.m.kc.Decode(key) @@ -176,7 +182,7 @@ func (c collectionImpl[K, V]) decodeKVPair(key, value []byte, delete bool) (inde } var v any - if decodeAny, ok := c.m.vc.(IndexableCodec); ok { + if decodeAny, ok := c.m.vc.(codec.IndexableCodec); ok { v, err = decodeAny.DecodeIndexable(value) } else { v, err = c.m.vc.Decode(value) @@ -193,8 +199,3 @@ func (c collectionImpl[K, V]) decodeKVPair(key, value []byte, delete bool) (inde Value: v, }, true, nil } - -type IndexableCodec interface { - SchemaFields() []indexerbase.Field - DecodeIndexable([]byte) (any, error) -} diff --git a/collections/map.go b/collections/map.go index 0b9b247aa27a..dec5bd19ebfa 100644 --- a/collections/map.go +++ b/collections/map.go @@ -17,9 +17,10 @@ type Map[K, V any] struct { vc codec.ValueCodec[V] // store accessor - sa func(context.Context) store.KVStore - prefix []byte - name string + sa func(context.Context) store.KVStore + prefix []byte + name string + isSecondaryIndex bool } // NewMap returns a Map given a StoreKey, a Prefix, human-readable name and the relative value and key encoders. From 1e1ffb72cbf0593853a97c4db8e871096af93f1d Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 17:21:34 -0400 Subject: [PATCH 42/63] refactor into standalone []Field validation methods --- indexer/base/fields.go | 71 +++++++++++++++ indexer/base/fields_test.go | 143 +++++++++++++++++++++++++++++++ indexer/base/object_type.go | 72 +--------------- indexer/base/object_type_test.go | 141 ------------------------------ 4 files changed, 216 insertions(+), 211 deletions(-) create mode 100644 indexer/base/fields.go create mode 100644 indexer/base/fields_test.go diff --git a/indexer/base/fields.go b/indexer/base/fields.go new file mode 100644 index 000000000000..84c051c58b82 --- /dev/null +++ b/indexer/base/fields.go @@ -0,0 +1,71 @@ +package indexerbase + +import "fmt" + +// ValidateWithKeyFields validates that the value conforms to the set of fields as a Key in an ObjectUpdate. +// See ObjectUpdate.Key for documentation on the requirements of such keys. +func ValidateWithKeyFields(keyFields []Field, value interface{}) error { + return validateFieldsValue(keyFields, value) +} + +// ValidateWithValueFields validates that the value conforms to the set of fields as a Value in an ObjectUpdate. +// See ObjectUpdate.Value for documentation on the requirements of such values. +func ValidateWithValueFields(valueFields []Field, value interface{}) error { + valueUpdates, ok := value.(ValueUpdates) + if !ok { + return validateFieldsValue(valueFields, value) + } + + values := map[string]interface{}{} + err := valueUpdates.Iterate(func(fieldname string, value interface{}) bool { + values[fieldname] = value + return true + }) + if err != nil { + return err + } + + for _, field := range valueFields { + v, ok := values[field.Name] + if !ok { + continue + } + + if err := field.ValidateValue(v); err != nil { + return err + } + + delete(values, field.Name) + } + + if len(values) > 0 { + return fmt.Errorf("unexpected values in ValueUpdates: %v", values) + } + + return nil +} + +func validateFieldsValue(fields []Field, value interface{}) error { + if len(fields) == 0 { + return nil + } + + if len(fields) == 1 { + return fields[0].ValidateValue(value) + } + + values, ok := value.([]interface{}) + if !ok { + return fmt.Errorf("expected slice of values for key fields, got %T", value) + } + + if len(fields) != len(values) { + return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]interface{}))) + } + for i, field := range fields { + if err := field.ValidateValue(values[i]); err != nil { + return err + } + } + return nil +} diff --git a/indexer/base/fields_test.go b/indexer/base/fields_test.go new file mode 100644 index 000000000000..beba2b24c34a --- /dev/null +++ b/indexer/base/fields_test.go @@ -0,0 +1,143 @@ +package indexerbase + +import ( + "strings" + "testing" +) + +func TestValidateWithKeyFields(t *testing.T) { + tests := []struct { + name string + keyFields []Field + key interface{} + errContains string + }{ + { + name: "no key fields", + keyFields: nil, + key: nil, + }, + { + name: "single key field, valid", + keyFields: object1Type.KeyFields, + key: "hello", + errContains: "", + }, + { + name: "single key field, invalid", + keyFields: object1Type.KeyFields, + key: []interface{}{"value"}, + errContains: "invalid value", + }, + { + name: "multiple key fields, valid", + keyFields: object2Type.KeyFields, + key: []interface{}{"hello", int32(42)}, + }, + { + name: "multiple key fields, not a slice", + keyFields: object2Type.KeyFields, + key: map[string]interface{}{"field1": "hello", "field2": "42"}, + errContains: "expected slice of values", + }, + { + name: "multiple key fields, wrong number of values", + keyFields: object2Type.KeyFields, + key: []interface{}{"hello"}, + errContains: "expected 2 key fields", + }, + { + name: "multiple key fields, invalid value", + keyFields: object2Type.KeyFields, + key: []interface{}{"hello", "abc"}, + errContains: "invalid value", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateWithKeyFields(tt.keyFields, tt.key) + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} + +func TestValidateWithValueFields(t *testing.T) { + tests := []struct { + name string + valueFields []Field + value interface{} + errContains string + }{ + { + name: "no value fields", + valueFields: nil, + value: nil, + }, + { + name: "single value field, valid", + valueFields: []Field{ + { + Name: "field1", + Kind: StringKind, + }, + }, + value: "hello", + errContains: "", + }, + { + name: "value updates, empty", + valueFields: object3Type.ValueFields, + value: MapValueUpdates(map[string]interface{}{}), + }, + { + name: "value updates, 1 field valid", + valueFields: object3Type.ValueFields, + value: MapValueUpdates(map[string]interface{}{ + "field1": "hello", + }), + }, + { + name: "value updates, 2 fields, 1 invalid", + valueFields: object3Type.ValueFields, + value: MapValueUpdates(map[string]interface{}{ + "field1": "hello", + "field2": "abc", + }), + errContains: "expected int32", + }, + { + name: "value updates, extra value", + valueFields: object3Type.ValueFields, + value: MapValueUpdates(map[string]interface{}{ + "field1": "hello", + "field2": int32(42), + "field3": "extra", + }), + errContains: "unexpected values", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateWithValueFields(tt.valueFields, tt.value) + if tt.errContains == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } else { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) + } + } + }) + } +} diff --git a/indexer/base/object_type.go b/indexer/base/object_type.go index 3b0db5afe8eb..52948ea1469a 100644 --- a/indexer/base/object_type.go +++ b/indexer/base/object_type.go @@ -56,7 +56,7 @@ func (o ObjectType) ValidateObjectUpdate(update ObjectUpdate) error { return fmt.Errorf("object type name %q does not match update type name %q", o.Name, update.TypeName) } - if err := o.ValidateKey(update.Key); err != nil { + if err := ValidateWithKeyFields(o.KeyFields, update.Key); err != nil { return fmt.Errorf("invalid key for object type %q: %w", update.TypeName, err) } @@ -64,73 +64,5 @@ func (o ObjectType) ValidateObjectUpdate(update ObjectUpdate) error { return nil } - return o.ValidateValue(update.Value) -} - -// ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate. -// See EntityUpdate.Key for documentation on the requirements of such keys. -func (o ObjectType) ValidateKey(value interface{}) error { - return validateFieldsValue(o.KeyFields, value) -} - -// ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate. -// See EntityUpdate.Value for documentation on the requirements of such values. -func (o ObjectType) ValidateValue(value interface{}) error { - valueUpdates, ok := value.(ValueUpdates) - if !ok { - return validateFieldsValue(o.ValueFields, value) - } - - values := map[string]interface{}{} - err := valueUpdates.Iterate(func(fieldname string, value interface{}) bool { - values[fieldname] = value - return true - }) - if err != nil { - return err - } - - for _, field := range o.ValueFields { - v, ok := values[field.Name] - if !ok { - continue - } - - if err := field.ValidateValue(v); err != nil { - return err - } - - delete(values, field.Name) - } - - if len(values) > 0 { - return fmt.Errorf("unexpected values in ValueUpdates: %v", values) - } - - return nil -} - -func validateFieldsValue(fields []Field, value interface{}) error { - if len(fields) == 0 { - return nil - } - - if len(fields) == 1 { - return fields[0].ValidateValue(value) - } - - values, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected slice of values for key fields, got %T", value) - } - - if len(fields) != len(values) { - return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]interface{}))) - } - for i, field := range fields { - if err := field.ValidateValue(values[i]); err != nil { - return err - } - } - return nil + return ValidateWithValueFields(o.ValueFields, update.Value) } diff --git a/indexer/base/object_type_test.go b/indexer/base/object_type_test.go index 89aab6dbf6c0..1c0d6736ec8d 100644 --- a/indexer/base/object_type_test.go +++ b/indexer/base/object_type_test.go @@ -130,147 +130,6 @@ func TestObjectType_Validate(t *testing.T) { } } -func TestObjectType_ValidateKey(t *testing.T) { - tests := []struct { - name string - objectType ObjectType - key interface{} - errContains string - }{ - { - name: "no key fields", - objectType: ObjectType{ - Name: "object1", - }, - key: nil, - }, - { - name: "single key field, valid", - objectType: object1Type, - key: "hello", - errContains: "", - }, - { - name: "single key field, invalid", - objectType: object1Type, - key: []interface{}{"value"}, - errContains: "invalid value", - }, - { - name: "multiple key fields, valid", - objectType: object2Type, - key: []interface{}{"hello", int32(42)}, - }, - { - name: "multiple key fields, not a slice", - objectType: object2Type, - key: map[string]interface{}{"field1": "hello", "field2": "42"}, - errContains: "expected slice of values", - }, - { - name: "multiple key fields, wrong number of values", - objectType: object2Type, - key: []interface{}{"hello"}, - errContains: "expected 2 key fields", - }, - { - name: "multiple key fields, invalid value", - objectType: object2Type, - key: []interface{}{"hello", "abc"}, - errContains: "invalid value", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.objectType.ValidateKey(tt.key) - if tt.errContains == "" { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } else { - if err == nil || !strings.Contains(err.Error(), tt.errContains) { - t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) - } - } - }) - } -} - -func TestObjectType_ValidateValue(t *testing.T) { - tests := []struct { - name string - objectType ObjectType - value interface{} - errContains string - }{ - { - name: "no value fields", - objectType: ObjectType{Name: "object0"}, - value: nil, - }, - { - name: "single value field, valid", - objectType: ObjectType{ - ValueFields: []Field{ - { - Name: "field1", - Kind: StringKind, - }, - }, - }, - value: "hello", - errContains: "", - }, - { - name: "value updates, empty", - objectType: object3Type, - value: MapValueUpdates(map[string]interface{}{}), - }, - { - name: "value updates, 1 field valid", - objectType: object3Type, - value: MapValueUpdates(map[string]interface{}{ - "field1": "hello", - }), - }, - { - name: "value updates, 2 fields, 1 invalid", - objectType: object3Type, - value: MapValueUpdates(map[string]interface{}{ - "field1": "hello", - "field2": "abc", - }), - errContains: "expected int32", - }, - { - name: "value updates, extra value", - objectType: object3Type, - value: MapValueUpdates(map[string]interface{}{ - "field1": "hello", - "field2": int32(42), - "field3": "extra", - }), - errContains: "unexpected values", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.objectType.ValidateValue(tt.value) - if tt.errContains == "" { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } else { - if err == nil || !strings.Contains(err.Error(), tt.errContains) { - t.Fatalf("expected error to contain %q, got: %v", tt.errContains, err) - } - } - }) - } -} - func TestObjectType_ValidateObjectUpdate(t *testing.T) { tests := []struct { name string From 0ab14f05d0f1a8e92593e72725fa107e3ce24a7c Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 18 Jun 2024 17:54:20 -0400 Subject: [PATCH 43/63] updates --- collections/codec/codec.go | 13 ++- collections/collections.go | 10 +- collections/indexing.go | 201 ++++++++++++++++++------------------- 3 files changed, 111 insertions(+), 113 deletions(-) diff --git a/collections/codec/codec.go b/collections/codec/codec.go index 6f58c31e3469..79dd90cb59e8 100644 --- a/collections/codec/codec.go +++ b/collections/codec/codec.go @@ -80,13 +80,16 @@ type ValueCodec[T any] interface { // It is not required by KeyCodec or ValueCodec in order to preserve backwards compatibility, but // a future version of collections may make it required and all codec's should aim to implement it. // If it is not implemented, fallback defaults will be used for indexing that may be sub-optimal. +// +// Implementations of IndexableCodec should test that they are conformant using the indexerbase.ValidateWithKeyFields +// and indexerbase.ValidateWithValueFields depending on whether the codec is a KeyCodec or ValueCodec respectively. type IndexableCodec interface { + LogicalDecoder() LogicalDecoder +} - // SchemaFields returns the logical schema fields of the codec's type. - SchemaFields() []indexerbase.Field - - // DecodeIndexable decodes the provided bytes into an indexable value that conforms to the codec's schema fields. - DecodeIndexable([]byte) (any, error) +type LogicalDecoder struct { + Fields []indexerbase.Field + Decode func([]byte) (any, error) } // NewUntypedValueCodec returns an UntypedValueCodec for the provided ValueCodec. diff --git a/collections/collections.go b/collections/collections.go index f56ae1bbae3d..a75a8b95ace8 100644 --- a/collections/collections.go +++ b/collections/collections.go @@ -92,13 +92,17 @@ type Collection interface { genesisHandler - objectType() indexerbase.ObjectType - - decodeKVPair(key, value []byte, delete bool) (indexerbase.ObjectUpdate, bool, error) + logicalDecoder() logicalDecoder isSecondaryIndex() bool } +type logicalDecoder struct { + objectType indexerbase.ObjectType + keyDecoder func([]byte) (any, error) + valueDecoder func([]byte) (any, error) +} + // Prefix defines a segregation bytes namespace for specific collections objects. type Prefix []byte diff --git a/collections/indexing.go b/collections/indexing.go index 87a0ebfea4b5..8666e0611d79 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -2,6 +2,7 @@ package collections import ( "bytes" + "encoding/json" "fmt" "strings" @@ -17,20 +18,36 @@ type IndexingOptions struct { func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, error) { decoder := moduleDecoder{ - lookup: &btree.Map[string, *collDecoder]{}, + collectionLookup: &btree.Map[string, *collDecoder]{}, + } + + retainDeletions := make(map[string]bool) + for _, collName := range opts.RetainDeletionsFor { + retainDeletions[collName] = true } var objectTypes []indexerbase.ObjectType for _, collName := range s.collectionsOrdered { coll := s.collectionsByName[collName] + + // skip secondary indexes if coll.isSecondaryIndex() { continue } - schema := coll.objectType() - objectTypes = append(objectTypes, schema) - decoder.lookup.Set(string(coll.GetPrefix()), &collDecoder{Collection: coll}) + ld := coll.logicalDecoder() + + if !retainDeletions[coll.GetName()] { + ld.objectType.RetainDeletions = true + } + + objectTypes = append(objectTypes, ld.objectType) + decoder.collectionLookup.Set(string(coll.GetPrefix()), &collDecoder{ + Collection: coll, + logicalDecoder: ld, + }) } + return indexerbase.ModuleDecoder{ Schema: indexerbase.ModuleSchema{ ObjectTypes: objectTypes, @@ -40,13 +57,13 @@ func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, } type moduleDecoder struct { - lookup *btree.Map[string, *collDecoder] + collectionLookup *btree.Map[string, *collDecoder] } func (m moduleDecoder) decodeKV(key, value []byte) (indexerbase.ObjectUpdate, bool, error) { ks := string(key) var cd *collDecoder - m.lookup.Descend(ks, func(prefix string, cur *collDecoder) bool { + m.collectionLookup.Descend(ks, func(prefix string, cur *collDecoder) bool { bytesPrefix := cur.GetPrefix() if bytes.HasPrefix(key, bytesPrefix) { cd = cur @@ -63,72 +80,94 @@ func (m moduleDecoder) decodeKV(key, value []byte) (indexerbase.ObjectUpdate, bo type collDecoder struct { Collection + logicalDecoder } -//type moduleStateDecoder struct { -// schema Schema -// collectionsIndex btree.BTree -//} -// -//func (m moduleStateDecoder) getCollectionForKey(key []byte) Collection { -// panic("implement me") -//} -// -//func (m moduleStateDecoder) DecodeSet(key, value []byte) (indexerbase.EntityUpdate, bool, error) { -// coll := m.getCollectionForKey(key) -// if coll == nil { -// return indexerbase.EntityUpdate{}, false, nil -// } -// -// return coll.decodeKVPair(key, value) -//} -// -//func (m moduleStateDecoder) DecodeDelete(key []byte) (indexerbase.EntityDelete, bool, error) { -// coll := m.getCollectionForKey(key) -// return coll.decodeDelete(key) -//} - -func (c collectionImpl[K, V]) getTableSchema() indexerbase.ObjectType { - var keyFields []indexerbase.Field - var valueFields []indexerbase.Field - - if hasSchema, ok := c.m.kc.(codec.IndexableCodec); ok { - keyFields = hasSchema.SchemaFields() - } else { - var k K - keyFields, _ = extractFields(k) +func (c collDecoder) decodeKVPair(key, value []byte, delete bool) (indexerbase.ObjectUpdate, bool, error) { + // strip prefix + key = key[len(c.GetPrefix()):] + + k, err := c.keyDecoder(key) + if err != nil { + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + }, true, err + } - ensureNames(c.m.kc, "key", keyFields) - if hasSchema, ok := c.m.vc.(codec.IndexableCodec); ok { - valueFields = hasSchema.SchemaFields() - } else { - var v V - valueFields, _ = extractFields(v) + if delete { + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + Key: k, + Delete: true, + }, true, nil } - ensureNames(c.m.vc, "value", valueFields) - return indexerbase.ObjectType{ - Name: c.GetName(), - KeyFields: keyFields, - ValueFields: valueFields, + v, err := c.valueDecoder(key) + if err != nil { + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + Key: k, + }, true, err + } + + return indexerbase.ObjectUpdate{ + TypeName: c.GetName(), + Key: k, + Value: v, + }, true, nil } -func extractFields(x any) ([]indexerbase.Field, func(any) any) { - if hasSchema, ok := x.(codec.IndexableCodec); ok { - return hasSchema.SchemaFields(), nil +func (c collectionImpl[K, V]) logicalDecoder() logicalDecoder { + res := logicalDecoder{} + + if indexable, ok := c.m.kc.(codec.IndexableCodec); ok { + keyDecoder := indexable.LogicalDecoder() + res.objectType.KeyFields = keyDecoder.Fields + res.keyDecoder = keyDecoder.Decode + } else { + fields, decoder := fallbackDecoder[K](func(bz []byte) (any, error) { + _, k, err := c.m.kc.Decode(bz) + return k, err + }) + res.objectType.KeyFields = fields + res.keyDecoder = decoder } + ensureFieldNames(c.m.kc, "key", res.objectType.KeyFields) - ty := indexerbase.KindForGoValue(x) - if ty > 0 { - return []indexerbase.Field{{Kind: ty}}, nil + if indexable, ok := c.m.vc.(codec.IndexableCodec); ok { + valueDecoder := indexable.LogicalDecoder() + res.objectType.KeyFields = valueDecoder.Fields + res.valueDecoder = valueDecoder.Decode + } else { + fields, decoder := fallbackDecoder[V](func(bz []byte) (any, error) { + v, err := c.m.vc.Decode(bz) + return v, err + }) + res.objectType.ValueFields = fields + res.valueDecoder = decoder } + ensureFieldNames(c.m.vc, "value", res.objectType.ValueFields) + + return res +} - panic(fmt.Errorf("unsupported type %T", x)) +func fallbackDecoder[T any](decode func([]byte) (any, error)) ([]indexerbase.Field, func([]byte) (any, error)) { + var t T + kind := indexerbase.KindForGoValue(t) + if err := kind.Validate(); err == nil { + return []indexerbase.Field{{Kind: kind}}, decode + } else { + return []indexerbase.Field{{Kind: indexerbase.JSONKind}}, func(b []byte) (any, error) { + t, err := decode(b) + bz, err := json.Marshal(t) + return json.RawMessage(bz), err + } + } } -func ensureNames(x any, defaultName string, cols []indexerbase.Field) { +func ensureFieldNames(x any, defaultName string, cols []indexerbase.Field) { var names []string = nil if hasName, ok := x.(interface{ Name() string }); ok { name := hasName.Name() @@ -151,51 +190,3 @@ func ensureNames(x any, defaultName string, cols []indexerbase.Field) { cols[i] = col } } - -func (c collectionImpl[K, V]) objectType() indexerbase.ObjectType { - //TODO implement me - panic("implement me") -} - -func (c collectionImpl[K, V]) decodeKVPair(key, value []byte, delete bool) (indexerbase.ObjectUpdate, bool, error) { - // strip prefix - key = key[len(c.GetPrefix()):] - var k any - var err error - if decodeAny, ok := c.m.kc.(codec.IndexableCodec); ok { - k, err = decodeAny.DecodeIndexable(key) - } else { - _, k, err = c.m.kc.Decode(key) - } - if err != nil { - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - }, false, err - } - - if delete { - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - Key: k, - Delete: true, - }, true, nil - } - - var v any - if decodeAny, ok := c.m.vc.(codec.IndexableCodec); ok { - v, err = decodeAny.DecodeIndexable(value) - } else { - v, err = c.m.vc.Decode(value) - } - if err != nil { - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - }, false, err - } - - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - Key: k, - Value: v, - }, true, nil -} From 2695659105a50987de63bbfc50e7d66673c21fda Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 3 Jul 2024 20:43:38 +0200 Subject: [PATCH 44/63] merge wip updates --- collections/codec/alternative_value_test.go | 2 +- collections/codec/bool.go | 15 +- collections/codec/bytes.go | 15 +- collections/codec/codec.go | 16 +-- collections/codec/int.go | 30 +++- collections/codec/naming.go | 35 +++++ collections/codec/string.go | 15 +- collections/codec/uint.go | 45 +++++- collections/collections.go | 7 +- collections/go.mod | 6 +- collections/indexing.go | 149 ++++++++++++-------- collections/naming_test.go | 30 ++++ collections/pair.go | 81 ++++++++++- collections/triple.go | 55 +++++++- 14 files changed, 403 insertions(+), 98 deletions(-) create mode 100644 collections/codec/naming.go create mode 100644 collections/naming_test.go diff --git a/collections/codec/alternative_value_test.go b/collections/codec/alternative_value_test.go index 358395427b90..871d95d653ce 100644 --- a/collections/codec/alternative_value_test.go +++ b/collections/codec/alternative_value_test.go @@ -17,7 +17,7 @@ type altValue struct { func TestAltValueCodec(t *testing.T) { // we assume we want to migrate the value from json(altValue) to just be // the raw value uint64. - canonical := codec.KeyToValueCodec(codec.NewUint64Key[uint64]()) + canonical := codec.KeyToValueCodec(codec.KeyCodec[uint64](codec.NewUint64Key[uint64]())) alternative := func(v []byte) (uint64, error) { var alt altValue err := json.Unmarshal(v, &alt) diff --git a/collections/codec/bool.go b/collections/codec/bool.go index 827af36c0715..f5a4616ca00b 100644 --- a/collections/codec/bool.go +++ b/collections/codec/bool.go @@ -6,9 +6,11 @@ import ( "strconv" ) -func NewBoolKey[T ~bool]() KeyCodec[T] { return boolKey[T]{} } +func NewBoolKey[T ~bool]() NameableKeyCodec[T] { return boolKey[T]{} } -type boolKey[T ~bool] struct{} +type boolKey[T ~bool] struct { + name string +} func (b boolKey[T]) Encode(buffer []byte, key T) (int, error) { if key { @@ -64,3 +66,12 @@ func (b boolKey[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { func (b boolKey[T]) SizeNonTerminal(key T) int { return b.Size(key) } + +func (b boolKey[T]) WithName(name string) NamedKeyCodec[T] { + b.name = name + return b +} + +func (b boolKey[T]) Name() string { + return b.name +} diff --git a/collections/codec/bytes.go b/collections/codec/bytes.go index 28334795e365..87b42046bcb0 100644 --- a/collections/codec/bytes.go +++ b/collections/codec/bytes.go @@ -10,9 +10,11 @@ import ( // using the BytesKey KeyCodec. const MaxBytesKeyNonTerminalSize = math.MaxUint8 -func NewBytesKey[T ~[]byte]() KeyCodec[T] { return bytesKey[T]{} } +func NewBytesKey[T ~[]byte]() NameableKeyCodec[T] { return bytesKey[T]{} } -type bytesKey[T ~[]byte] struct{} +type bytesKey[T ~[]byte] struct { + name string +} func (b bytesKey[T]) Encode(buffer []byte, key T) (int, error) { return copy(buffer, key), nil @@ -77,3 +79,12 @@ func (bytesKey[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { func (bytesKey[T]) SizeNonTerminal(key T) int { return len(key) + 1 } + +func (b bytesKey[T]) WithName(name string) NamedKeyCodec[T] { + b.name = name + return b +} + +func (b bytesKey[T]) Name() string { + return b.name +} diff --git a/collections/codec/codec.go b/collections/codec/codec.go index 79dd90cb59e8..83fe732bd55c 100644 --- a/collections/codec/codec.go +++ b/collections/codec/codec.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - indexerbase "cosmossdk.io/indexer/base" + "cosmossdk.io/schema" ) var ErrEncoding = errors.New("collections: encoding error") @@ -81,15 +81,15 @@ type ValueCodec[T any] interface { // a future version of collections may make it required and all codec's should aim to implement it. // If it is not implemented, fallback defaults will be used for indexing that may be sub-optimal. // -// Implementations of IndexableCodec should test that they are conformant using the indexerbase.ValidateWithKeyFields -// and indexerbase.ValidateWithValueFields depending on whether the codec is a KeyCodec or ValueCodec respectively. -type IndexableCodec interface { - LogicalDecoder() LogicalDecoder +// Implementations of IndexableCodec should test that they are conformant using the schema.ValidateWithKeyFields +// and schema.ValidateWithValueFields depending on whether the codec is a KeyCodec or ValueCodec respectively. +type IndexableCodec[T any] interface { + LogicalDecoder() (LogicalDecoder[T], error) } -type LogicalDecoder struct { - Fields []indexerbase.Field - Decode func([]byte) (any, error) +type LogicalDecoder[T any] struct { + Fields []schema.Field + ToSchemaType func(T) (any, error) } // NewUntypedValueCodec returns an UntypedValueCodec for the provided ValueCodec. diff --git a/collections/codec/int.go b/collections/codec/int.go index 1300efde4df0..5ee3e6fd6d6e 100644 --- a/collections/codec/int.go +++ b/collections/codec/int.go @@ -7,9 +7,11 @@ import ( "strconv" ) -func NewInt64Key[T ~int64]() KeyCodec[T] { return int64Key[T]{} } +func NewInt64Key[T ~int64]() NameableKeyCodec[T] { return int64Key[T]{} } -type int64Key[T ~int64] struct{} +type int64Key[T ~int64] struct { + name string +} func (i int64Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint64(buffer, (uint64)(key)) @@ -64,11 +66,22 @@ func (i int64Key[T]) SizeNonTerminal(_ T) int { return 8 } -func NewInt32Key[T ~int32]() KeyCodec[T] { +func (i int64Key[T]) WithName(name string) NamedKeyCodec[T] { + i.name = name + return i +} + +func (i int64Key[T]) Name() string { + return i.name +} + +func NewInt32Key[T ~int32]() NameableKeyCodec[T] { return int32Key[T]{} } -type int32Key[T ~int32] struct{} +type int32Key[T ~int32] struct { + name string +} func (i int32Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint32(buffer, (uint32)(key)) @@ -121,3 +134,12 @@ func (i int32Key[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { func (i int32Key[T]) SizeNonTerminal(_ T) int { return 4 } + +func (i int32Key[T]) WithName(name string) NamedKeyCodec[T] { + i.name = name + return i +} + +func (i int32Key[T]) Name() string { + return i.name +} diff --git a/collections/codec/naming.go b/collections/codec/naming.go new file mode 100644 index 000000000000..fe60ee0e8cae --- /dev/null +++ b/collections/codec/naming.go @@ -0,0 +1,35 @@ +package codec + +type HasName interface { + // Name returns the name of key in the schema if one is defined or the empty string. + // Multipart keys should separate names with commas, i.e. "name1,name2". + Name() string +} + +// NameableKeyCodec is a KeyCodec that can be named. +type NameableKeyCodec[T any] interface { + KeyCodec[T] + + // WithName returns the KeyCodec with the provided name. + WithName(name string) NamedKeyCodec[T] +} + +// NamedKeyCodec is a KeyCodec that has a name. +type NamedKeyCodec[T any] interface { + KeyCodec[T] + HasName +} + +// NameableValueCodec is a ValueCodec that can be named. +type NameableValueCodec[T any] interface { + ValueCodec[T] + + // WithName returns the ValueCodec with the provided name. + WithName(name string) NamedValueCodec[T] +} + +// NamedValueCodec is a ValueCodec that has a name. +type NamedValueCodec[T any] interface { + ValueCodec[T] + HasName +} diff --git a/collections/codec/string.go b/collections/codec/string.go index 3189b8bc9cbf..3c9d53b1c42b 100644 --- a/collections/codec/string.go +++ b/collections/codec/string.go @@ -6,14 +6,16 @@ import ( "fmt" ) -func NewStringKeyCodec[T ~string]() KeyCodec[T] { return stringKey[T]{} } +func NewStringKeyCodec[T ~string]() NameableKeyCodec[T] { return stringKey[T]{} } const ( // StringDelimiter defines the delimiter of a string key when used in non-terminal encodings. StringDelimiter uint8 = 0x0 ) -type stringKey[T ~string] struct{} +type stringKey[T ~string] struct { + name string +} func (stringKey[T]) Encode(buffer []byte, key T) (int, error) { return copy(buffer, key), nil @@ -66,3 +68,12 @@ func (stringKey[T]) Stringify(key T) string { func (stringKey[T]) KeyType() string { return "string" } + +func (s stringKey[T]) WithName(name string) NamedKeyCodec[T] { + s.name = name + return s +} + +func (s stringKey[T]) Name() string { + return s.name +} diff --git a/collections/codec/uint.go b/collections/codec/uint.go index 658235d385ad..8586aeb1b6ff 100644 --- a/collections/codec/uint.go +++ b/collections/codec/uint.go @@ -7,9 +7,11 @@ import ( "strconv" ) -func NewUint64Key[T ~uint64]() KeyCodec[T] { return uint64Key[T]{} } +func NewUint64Key[T ~uint64]() NameableKeyCodec[T] { return uint64Key[T]{} } -type uint64Key[T ~uint64] struct{} +type uint64Key[T ~uint64] struct { + name string +} func (uint64Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint64(buffer, (uint64)(key)) @@ -55,9 +57,20 @@ func (uint64Key[T]) KeyType() string { return "uint64" } -func NewUint32Key[T ~uint32]() KeyCodec[T] { return uint32Key[T]{} } +func (u uint64Key[T]) WithName(name string) NamedKeyCodec[T] { + u.name = name + return u +} + +func (u uint64Key[T]) Name() string { + return u.name +} + +func NewUint32Key[T ~uint32]() NameableKeyCodec[T] { return uint32Key[T]{} } -type uint32Key[T ~uint32] struct{} +type uint32Key[T ~uint32] struct { + name string +} func (uint32Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint32(buffer, (uint32)(key)) @@ -95,9 +108,20 @@ func (u uint32Key[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { return func (uint32Key[T]) SizeNonTerminal(_ T) int { return 4 } -func NewUint16Key[T ~uint16]() KeyCodec[T] { return uint16Key[T]{} } +func (u uint32Key[T]) WithName(name string) NamedKeyCodec[T] { + u.name = name + return u +} -type uint16Key[T ~uint16] struct{} +func (u uint32Key[T]) Name() string { + return u.name +} + +func NewUint16Key[T ~uint16]() NameableKeyCodec[T] { return uint16Key[T]{} } + +type uint16Key[T ~uint16] struct { + name string +} func (uint16Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint16(buffer, (uint16)(key)) @@ -135,6 +159,15 @@ func (u uint16Key[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { return func (u uint16Key[T]) SizeNonTerminal(key T) int { return u.Size(key) } +func (u uint16Key[T]) WithName(name string) NamedKeyCodec[T] { + u.name = name + return u +} + +func (u uint16Key[T]) Name() string { + return u.name +} + func uintEncodeJSON(value uint64) ([]byte, error) { str := `"` + strconv.FormatUint(value, 10) + `"` return []byte(str), nil diff --git a/collections/collections.go b/collections/collections.go index a75a8b95ace8..05bfcca2a2be 100644 --- a/collections/collections.go +++ b/collections/collections.go @@ -6,8 +6,9 @@ import ( "io" "math" + "cosmossdk.io/schema" + "cosmossdk.io/collections/codec" - indexerbase "cosmossdk.io/indexer/base" ) var ( @@ -92,13 +93,13 @@ type Collection interface { genesisHandler - logicalDecoder() logicalDecoder + logicalDecoder() (logicalDecoder, error) isSecondaryIndex() bool } type logicalDecoder struct { - objectType indexerbase.ObjectType + objectType schema.ObjectType keyDecoder func([]byte) (any, error) valueDecoder func([]byte) (any, error) } diff --git a/collections/go.mod b/collections/go.mod index 626b082e0182..1cbf81c30423 100644 --- a/collections/go.mod +++ b/collections/go.mod @@ -5,9 +5,10 @@ go 1.21 require ( cosmossdk.io/core v0.12.0 cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 + cosmossdk.io/schema v0.0.0 github.com/stretchr/testify v1.9.0 + github.com/tidwall/btree v1.7.0 pgregory.net/rapid v1.1.0 - cosmossdk.io/indexer/base v0.0.0-00010101000000-000000000000 ) require ( @@ -17,7 +18,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/tidwall/btree v1.7.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect @@ -31,5 +31,5 @@ require ( replace ( cosmossdk.io/core => ../core cosmossdk.io/core/testing => ../core/testing - cosmossdk.io/indexer/base => ../indexer/base + cosmossdk.io/schema => ../schema ) diff --git a/collections/indexing.go b/collections/indexing.go index 8666e0611d79..e4529359090d 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -8,15 +8,17 @@ import ( "github.com/tidwall/btree" + "cosmossdk.io/schema" + "cosmossdk.io/collections/codec" - indexerbase "cosmossdk.io/indexer/base" ) type IndexingOptions struct { + // RetainDeletionsFor is the list of collections to retain deletions for. RetainDeletionsFor []string } -func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, error) { +func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { decoder := moduleDecoder{ collectionLookup: &btree.Map[string, *collDecoder]{}, } @@ -26,7 +28,7 @@ func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, retainDeletions[collName] = true } - var objectTypes []indexerbase.ObjectType + var objectTypes []schema.ObjectType for _, collName := range s.collectionsOrdered { coll := s.collectionsByName[collName] @@ -35,7 +37,10 @@ func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, continue } - ld := coll.logicalDecoder() + ld, err := coll.logicalDecoder() + if err != nil { + return schema.ModuleCodec{}, err + } if !retainDeletions[coll.GetName()] { ld.objectType.RetainDeletions = true @@ -48,8 +53,8 @@ func (s Schema) ModuleDecoder(opts IndexingOptions) (indexerbase.ModuleDecoder, }) } - return indexerbase.ModuleDecoder{ - Schema: indexerbase.ModuleSchema{ + return schema.ModuleCodec{ + Schema: schema.ModuleSchema{ ObjectTypes: objectTypes, }, KVDecoder: decoder.decodeKV, @@ -60,7 +65,8 @@ type moduleDecoder struct { collectionLookup *btree.Map[string, *collDecoder] } -func (m moduleDecoder) decodeKV(key, value []byte) (indexerbase.ObjectUpdate, bool, error) { +func (m moduleDecoder) decodeKV(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { + key := update.Key ks := string(key) var cd *collDecoder m.collectionLookup.Descend(ks, func(prefix string, cur *collDecoder) bool { @@ -72,10 +78,10 @@ func (m moduleDecoder) decodeKV(key, value []byte) (indexerbase.ObjectUpdate, bo return false }) if cd == nil { - return indexerbase.ObjectUpdate{}, false, nil + return nil, nil } - return cd.decodeKVPair(key, value, false) + return cd.decodeKVPair(update) } type collDecoder struct { @@ -83,91 +89,110 @@ type collDecoder struct { logicalDecoder } -func (c collDecoder) decodeKVPair(key, value []byte, delete bool) (indexerbase.ObjectUpdate, bool, error) { +func (c collDecoder) decodeKVPair(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { // strip prefix + key := update.Key key = key[len(c.GetPrefix()):] k, err := c.keyDecoder(key) if err != nil { - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - }, true, err + return []schema.ObjectUpdate{ + {TypeName: c.GetName()}, + }, err } - if delete { - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - Key: k, - Delete: true, - }, true, nil + if update.Delete { + return []schema.ObjectUpdate{ + {TypeName: c.GetName(), Key: k, Delete: true}, + }, nil } - v, err := c.valueDecoder(key) + v, err := c.valueDecoder(update.Value) if err != nil { - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - Key: k, - }, true, err - + return []schema.ObjectUpdate{ + {TypeName: c.GetName(), Key: k}, + }, err } - return indexerbase.ObjectUpdate{ - TypeName: c.GetName(), - Key: k, - Value: v, - }, true, nil + return []schema.ObjectUpdate{ + {TypeName: c.GetName(), Key: k, Value: v}, + }, nil } -func (c collectionImpl[K, V]) logicalDecoder() logicalDecoder { +func (c collectionImpl[K, V]) logicalDecoder() (logicalDecoder, error) { res := logicalDecoder{} + res.objectType.Name = c.GetName() - if indexable, ok := c.m.kc.(codec.IndexableCodec); ok { - keyDecoder := indexable.LogicalDecoder() - res.objectType.KeyFields = keyDecoder.Fields - res.keyDecoder = keyDecoder.Decode - } else { - fields, decoder := fallbackDecoder[K](func(bz []byte) (any, error) { - _, k, err := c.m.kc.Decode(bz) - return k, err - }) - res.objectType.KeyFields = fields - res.keyDecoder = decoder + keyDecoder, err := KeyCodecDecoder(c.m.kc) + if err != nil { + return logicalDecoder{}, err + } + res.objectType.KeyFields = keyDecoder.Fields + res.keyDecoder = func(i []byte) (any, error) { + _, x, err := c.m.kc.Decode(i) + if err != nil { + return nil, err + } + return keyDecoder.ToSchemaType(x) } ensureFieldNames(c.m.kc, "key", res.objectType.KeyFields) - if indexable, ok := c.m.vc.(codec.IndexableCodec); ok { - valueDecoder := indexable.LogicalDecoder() - res.objectType.KeyFields = valueDecoder.Fields - res.valueDecoder = valueDecoder.Decode - } else { - fields, decoder := fallbackDecoder[V](func(bz []byte) (any, error) { - v, err := c.m.vc.Decode(bz) - return v, err - }) - res.objectType.ValueFields = fields - res.valueDecoder = decoder + valueDecoder, err := ValueCodecDecoder(c.m.vc) + if err != nil { + return logicalDecoder{}, err + } + res.objectType.ValueFields = valueDecoder.Fields + res.valueDecoder = func(i []byte) (any, error) { + x, err := c.m.vc.Decode(i) + if err != nil { + return nil, err + } + return valueDecoder.ToSchemaType(x) } ensureFieldNames(c.m.vc, "value", res.objectType.ValueFields) - return res + return res, nil +} + +func KeyCodecDecoder[K any](cdc codec.KeyCodec[K]) (codec.LogicalDecoder[K], error) { + if indexable, ok := cdc.(codec.IndexableCodec[K]); ok { + return indexable.LogicalDecoder() + } else { + return FallbackDecoder[K](), nil + } } -func fallbackDecoder[T any](decode func([]byte) (any, error)) ([]indexerbase.Field, func([]byte) (any, error)) { +func ValueCodecDecoder[K any](cdc codec.ValueCodec[K]) (codec.LogicalDecoder[K], error) { + if indexable, ok := cdc.(codec.IndexableCodec[K]); ok { + return indexable.LogicalDecoder() + } else { + return FallbackDecoder[K](), nil + } +} + +func FallbackDecoder[T any]() codec.LogicalDecoder[T] { var t T - kind := indexerbase.KindForGoValue(t) + kind := schema.KindForGoValue(t) if err := kind.Validate(); err == nil { - return []indexerbase.Field{{Kind: kind}}, decode + return codec.LogicalDecoder[T]{ + Fields: []schema.Field{{Kind: kind}}, + ToSchemaType: func(t T) (any, error) { + return t, nil + }, + } } else { - return []indexerbase.Field{{Kind: indexerbase.JSONKind}}, func(b []byte) (any, error) { - t, err := decode(b) - bz, err := json.Marshal(t) - return json.RawMessage(bz), err + return codec.LogicalDecoder[T]{ + Fields: []schema.Field{{Kind: schema.JSONKind}}, + ToSchemaType: func(t T) (any, error) { + bz, err := json.Marshal(t) + return json.RawMessage(bz), err + }, } } } -func ensureFieldNames(x any, defaultName string, cols []indexerbase.Field) { +func ensureFieldNames(x any, defaultName string, cols []schema.Field) { var names []string = nil if hasName, ok := x.(interface{ Name() string }); ok { name := hasName.Name() diff --git a/collections/naming_test.go b/collections/naming_test.go new file mode 100644 index 000000000000..48d05d8d7670 --- /dev/null +++ b/collections/naming_test.go @@ -0,0 +1,30 @@ +package collections + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNaming(t *testing.T) { + require.Equal(t, "u16", Uint16Key.WithName("u16").Name()) + require.Equal(t, "u32", Uint32Key.WithName("u32").Name()) + require.Equal(t, "u64", Uint64Key.WithName("u64").Name()) + require.Equal(t, "i32", Int32Key.WithName("i32").Name()) + require.Equal(t, "i64", Int64Key.WithName("i64").Name()) + require.Equal(t, "str", StringKey.WithName("str").Name()) + require.Equal(t, "bytes", BytesKey.WithName("bytes").Name()) + require.Equal(t, "bool", BoolKey.WithName("bool").Name()) + + require.Equal(t, "vu16", Uint16Value.WithName("vu16").Name()) + require.Equal(t, "vu32", Uint32Value.WithName("vu32").Name()) + require.Equal(t, "vu64", Uint64Value.WithName("vu64").Name()) + require.Equal(t, "vi32", Int32Value.WithName("vi32").Name()) + require.Equal(t, "vi64", Int64Value.WithName("vi64").Name()) + require.Equal(t, "vstr", StringValue.WithName("vstr").Name()) + require.Equal(t, "vbytes", BytesValue.WithName("vbytes").Name()) + require.Equal(t, "vbool", BoolValue.WithName("vbool").Name()) + + require.Equal(t, "abc,def", NamedPairKeyCodec[bool, string]("abc", BoolKey, "def", StringKey).Name()) + require.Equal(t, "abc,def,ghi", NamedTripleKeyCodec[bool, string, int32]("abc", BoolKey, "def", StringKey, "ghi", Int32Key).Name()) +} diff --git a/collections/pair.go b/collections/pair.go index f12aaac1b576..4f0f91c0d552 100644 --- a/collections/pair.go +++ b/collections/pair.go @@ -6,6 +6,7 @@ import ( "strings" "cosmossdk.io/collections/codec" + "cosmossdk.io/schema" ) // Pair defines a key composed of two keys. @@ -54,9 +55,22 @@ func PairKeyCodec[K1, K2 any](keyCodec1 codec.KeyCodec[K1], keyCodec2 codec.KeyC } } +// NamedPairKeyCodec instantiates a new KeyCodec instance that can encode the Pair, given the KeyCodec of the +// first part of the key and the KeyCodec of the second part of the key, with names assigned to each part +// which will only be used for indexing and informational purposes. +func NamedPairKeyCodec[K1, K2 any](key1Name string, keyCodec1 codec.KeyCodec[K1], key2Name string, keyCodec2 codec.KeyCodec[K2]) codec.NamedKeyCodec[Pair[K1, K2]] { + return pairKeyCodec[K1, K2]{ + key1Name: key1Name, + key2Name: key2Name, + keyCodec1: keyCodec1, + keyCodec2: keyCodec2, + } +} + type pairKeyCodec[K1, K2 any] struct { - keyCodec1 codec.KeyCodec[K1] - keyCodec2 codec.KeyCodec[K2] + key1Name, key2Name string + keyCodec1 codec.KeyCodec[K1] + keyCodec2 codec.KeyCodec[K2] } func (p pairKeyCodec[K1, K2]) KeyCodec1() codec.KeyCodec[K1] { return p.keyCodec1 } @@ -216,6 +230,69 @@ func (p pairKeyCodec[K1, K2]) DecodeJSON(b []byte) (Pair[K1, K2], error) { return Join(k1, k2), nil } +func (p pairKeyCodec[K1, K2]) Name() string { + if p.key1Name == "" || p.key2Name == "" { + return "key1,key2" + } + return fmt.Sprintf("%s,%s", p.key1Name, p.key2Name) +} + +func (p pairKeyCodec[K1, K2]) LogicalDecoder() (res codec.LogicalDecoder[Pair[K1, K2]], err error) { + dec1, err := KeyCodecDecoder(p.keyCodec1) + if err != nil { + return + } + + dec2, err := KeyCodecDecoder(p.keyCodec2) + if err != nil { + return + } + + if len(dec1.Fields) != 1 { + err = fmt.Errorf("key1 codec must have exactly one field") + return + } + if len(dec2.Fields) != 1 { + err = fmt.Errorf("key1 codec must have exactly one field") + return + } + + fields := []schema.Field{dec1.Fields[0], dec2.Fields[0]} + fields[0].Name = p.key1Name + fields[1].Name = p.key2Name + + return codec.LogicalDecoder[Pair[K1, K2]]{ + Fields: fields, + ToSchemaType: func(pair Pair[K1, K2]) (any, error) { + k1, err := dec1.ToSchemaType(*pair.key1) + if err != nil { + return nil, err + } + + k2, err := dec2.ToSchemaType(*pair.key2) + if err != nil { + return nil, err + } + + return []any{k1, k2}, nil + }, + }, nil +} + +//func (p pairKeyCodec[K1, K2]) SchemaColumns() []schema.Field { +// //var k1 K1 +// //col1, _ := extractFields(k1) +// //if len(col1) == 1 { +// // col1[0].Name = p.key1Name +// //} +// //var k2 K2 +// //col2, _ := extractFields(k2) +// //if len(col2) == 1 { +// // col2[0].Name = p.key2Name +// //} +// //return append(col1, col2...) +//} + // NewPrefixUntilPairRange defines a collection query which ranges until the provided Pair prefix. // Unstable: this API might change in the future. func NewPrefixUntilPairRange[K1, K2 any](prefix K1) *PairRange[K1, K2] { diff --git a/collections/triple.go b/collections/triple.go index 9733d9984099..25ac7f59f782 100644 --- a/collections/triple.go +++ b/collections/triple.go @@ -64,10 +64,22 @@ func TripleKeyCodec[K1, K2, K3 any](keyCodec1 codec.KeyCodec[K1], keyCodec2 code } } +func NamedTripleKeyCodec[K1, K2, K3 any](key1Name string, keyCodec1 codec.KeyCodec[K1], key2Name string, keyCodec2 codec.KeyCodec[K2], key3Name string, keyCodec3 codec.KeyCodec[K3]) codec.NamedKeyCodec[Triple[K1, K2, K3]] { + return tripleKeyCodec[K1, K2, K3]{ + key1Name: key1Name, + key2Name: key2Name, + key3Name: key3Name, + keyCodec1: keyCodec1, + keyCodec2: keyCodec2, + keyCodec3: keyCodec3, + } +} + type tripleKeyCodec[K1, K2, K3 any] struct { - keyCodec1 codec.KeyCodec[K1] - keyCodec2 codec.KeyCodec[K2] - keyCodec3 codec.KeyCodec[K3] + key1Name, key2Name, key3Name string + keyCodec1 codec.KeyCodec[K1] + keyCodec2 codec.KeyCodec[K2] + keyCodec3 codec.KeyCodec[K3] } type jsonTripleKey [3]json.RawMessage @@ -273,6 +285,43 @@ func (t tripleKeyCodec[K1, K2, K3]) SizeNonTerminal(key Triple[K1, K2, K3]) int return size } +func (t tripleKeyCodec[K1, K2, K3]) Name() string { + if t.key1Name == "" || t.key2Name == "" || t.key3Name == "" { + return "key1,key2,key3" + } + return fmt.Sprintf("%s,%s,%s", t.key1Name, t.key2Name, t.key3Name) +} + +//func (p tripleKeyCodec[K1, K2, K3]) SchemaColumns() []indexerbase.Column { +// var k1 K1 +// col1, _ := extractFields(k1) +// if len(col1) == 1 { +// col1[0].Name = p.key1Name +// } +// var k2 K2 +// col2, _ := extractFields(k2) +// if len(col2) == 1 { +// col2[0].Name = p.key2Name +// } +// var k3 K3 +// col3, _ := extractFields(k3) +// if len(col3) == 1 { +// col3[0].Name = p.key3Name +// +// } +// cols := append(col1, col2...) +// cols = append(cols, col3...) +// return cols +//} +// +//func (p tripleKeyCodec[K1, K2, K3]) DecodeIndexable(buffer []byte) (any, error) { +// _, x, err := p.Decode(buffer) +// if err != nil { +// return nil, err +// } +// return []any{x.K1(), x.K2(), x.K3()}, nil +//} + // NewPrefixUntilTripleRange defines a collection query which ranges until the provided Pair prefix. // Unstable: this API might change in the future. func NewPrefixUntilTripleRange[K1, K2, K3 any](k1 K1) Ranger[Triple[K1, K2, K3]] { From 00b08d9834b0aa64951f2264005bd0b2070815e7 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 3 Jul 2024 20:44:15 +0200 Subject: [PATCH 45/63] revert --- schema/object_update.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/schema/object_update.go b/schema/object_update.go index ccc9c40405d4..455c4a850afb 100644 --- a/schema/object_update.go +++ b/schema/object_update.go @@ -2,8 +2,6 @@ package schema import "sort" -import "sort" - // ObjectUpdate represents an update operation on an object in a module's state. type ObjectUpdate struct { // TypeName is the name of the object type in the module's schema. From f45bcd7b04e94729134033e5da96f2cd2fdc66cb Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 8 Jul 2024 16:19:30 +0200 Subject: [PATCH 46/63] reduce PR scope --- collections/codec/alternative_value_test.go | 2 +- collections/codec/bool.go | 15 +--- collections/codec/bytes.go | 15 +--- collections/codec/int.go | 30 +------- collections/codec/naming.go | 35 --------- collections/codec/string.go | 15 +--- collections/codec/uint.go | 45 ++---------- collections/naming_test.go | 30 -------- collections/pair.go | 81 +-------------------- collections/triple.go | 55 +------------- 10 files changed, 22 insertions(+), 301 deletions(-) delete mode 100644 collections/codec/naming.go delete mode 100644 collections/naming_test.go diff --git a/collections/codec/alternative_value_test.go b/collections/codec/alternative_value_test.go index 871d95d653ce..358395427b90 100644 --- a/collections/codec/alternative_value_test.go +++ b/collections/codec/alternative_value_test.go @@ -17,7 +17,7 @@ type altValue struct { func TestAltValueCodec(t *testing.T) { // we assume we want to migrate the value from json(altValue) to just be // the raw value uint64. - canonical := codec.KeyToValueCodec(codec.KeyCodec[uint64](codec.NewUint64Key[uint64]())) + canonical := codec.KeyToValueCodec(codec.NewUint64Key[uint64]()) alternative := func(v []byte) (uint64, error) { var alt altValue err := json.Unmarshal(v, &alt) diff --git a/collections/codec/bool.go b/collections/codec/bool.go index f5a4616ca00b..827af36c0715 100644 --- a/collections/codec/bool.go +++ b/collections/codec/bool.go @@ -6,11 +6,9 @@ import ( "strconv" ) -func NewBoolKey[T ~bool]() NameableKeyCodec[T] { return boolKey[T]{} } +func NewBoolKey[T ~bool]() KeyCodec[T] { return boolKey[T]{} } -type boolKey[T ~bool] struct { - name string -} +type boolKey[T ~bool] struct{} func (b boolKey[T]) Encode(buffer []byte, key T) (int, error) { if key { @@ -66,12 +64,3 @@ func (b boolKey[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { func (b boolKey[T]) SizeNonTerminal(key T) int { return b.Size(key) } - -func (b boolKey[T]) WithName(name string) NamedKeyCodec[T] { - b.name = name - return b -} - -func (b boolKey[T]) Name() string { - return b.name -} diff --git a/collections/codec/bytes.go b/collections/codec/bytes.go index 87b42046bcb0..28334795e365 100644 --- a/collections/codec/bytes.go +++ b/collections/codec/bytes.go @@ -10,11 +10,9 @@ import ( // using the BytesKey KeyCodec. const MaxBytesKeyNonTerminalSize = math.MaxUint8 -func NewBytesKey[T ~[]byte]() NameableKeyCodec[T] { return bytesKey[T]{} } +func NewBytesKey[T ~[]byte]() KeyCodec[T] { return bytesKey[T]{} } -type bytesKey[T ~[]byte] struct { - name string -} +type bytesKey[T ~[]byte] struct{} func (b bytesKey[T]) Encode(buffer []byte, key T) (int, error) { return copy(buffer, key), nil @@ -79,12 +77,3 @@ func (bytesKey[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { func (bytesKey[T]) SizeNonTerminal(key T) int { return len(key) + 1 } - -func (b bytesKey[T]) WithName(name string) NamedKeyCodec[T] { - b.name = name - return b -} - -func (b bytesKey[T]) Name() string { - return b.name -} diff --git a/collections/codec/int.go b/collections/codec/int.go index 5ee3e6fd6d6e..1300efde4df0 100644 --- a/collections/codec/int.go +++ b/collections/codec/int.go @@ -7,11 +7,9 @@ import ( "strconv" ) -func NewInt64Key[T ~int64]() NameableKeyCodec[T] { return int64Key[T]{} } +func NewInt64Key[T ~int64]() KeyCodec[T] { return int64Key[T]{} } -type int64Key[T ~int64] struct { - name string -} +type int64Key[T ~int64] struct{} func (i int64Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint64(buffer, (uint64)(key)) @@ -66,22 +64,11 @@ func (i int64Key[T]) SizeNonTerminal(_ T) int { return 8 } -func (i int64Key[T]) WithName(name string) NamedKeyCodec[T] { - i.name = name - return i -} - -func (i int64Key[T]) Name() string { - return i.name -} - -func NewInt32Key[T ~int32]() NameableKeyCodec[T] { +func NewInt32Key[T ~int32]() KeyCodec[T] { return int32Key[T]{} } -type int32Key[T ~int32] struct { - name string -} +type int32Key[T ~int32] struct{} func (i int32Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint32(buffer, (uint32)(key)) @@ -134,12 +121,3 @@ func (i int32Key[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { func (i int32Key[T]) SizeNonTerminal(_ T) int { return 4 } - -func (i int32Key[T]) WithName(name string) NamedKeyCodec[T] { - i.name = name - return i -} - -func (i int32Key[T]) Name() string { - return i.name -} diff --git a/collections/codec/naming.go b/collections/codec/naming.go deleted file mode 100644 index fe60ee0e8cae..000000000000 --- a/collections/codec/naming.go +++ /dev/null @@ -1,35 +0,0 @@ -package codec - -type HasName interface { - // Name returns the name of key in the schema if one is defined or the empty string. - // Multipart keys should separate names with commas, i.e. "name1,name2". - Name() string -} - -// NameableKeyCodec is a KeyCodec that can be named. -type NameableKeyCodec[T any] interface { - KeyCodec[T] - - // WithName returns the KeyCodec with the provided name. - WithName(name string) NamedKeyCodec[T] -} - -// NamedKeyCodec is a KeyCodec that has a name. -type NamedKeyCodec[T any] interface { - KeyCodec[T] - HasName -} - -// NameableValueCodec is a ValueCodec that can be named. -type NameableValueCodec[T any] interface { - ValueCodec[T] - - // WithName returns the ValueCodec with the provided name. - WithName(name string) NamedValueCodec[T] -} - -// NamedValueCodec is a ValueCodec that has a name. -type NamedValueCodec[T any] interface { - ValueCodec[T] - HasName -} diff --git a/collections/codec/string.go b/collections/codec/string.go index 3c9d53b1c42b..3189b8bc9cbf 100644 --- a/collections/codec/string.go +++ b/collections/codec/string.go @@ -6,16 +6,14 @@ import ( "fmt" ) -func NewStringKeyCodec[T ~string]() NameableKeyCodec[T] { return stringKey[T]{} } +func NewStringKeyCodec[T ~string]() KeyCodec[T] { return stringKey[T]{} } const ( // StringDelimiter defines the delimiter of a string key when used in non-terminal encodings. StringDelimiter uint8 = 0x0 ) -type stringKey[T ~string] struct { - name string -} +type stringKey[T ~string] struct{} func (stringKey[T]) Encode(buffer []byte, key T) (int, error) { return copy(buffer, key), nil @@ -68,12 +66,3 @@ func (stringKey[T]) Stringify(key T) string { func (stringKey[T]) KeyType() string { return "string" } - -func (s stringKey[T]) WithName(name string) NamedKeyCodec[T] { - s.name = name - return s -} - -func (s stringKey[T]) Name() string { - return s.name -} diff --git a/collections/codec/uint.go b/collections/codec/uint.go index 8586aeb1b6ff..658235d385ad 100644 --- a/collections/codec/uint.go +++ b/collections/codec/uint.go @@ -7,11 +7,9 @@ import ( "strconv" ) -func NewUint64Key[T ~uint64]() NameableKeyCodec[T] { return uint64Key[T]{} } +func NewUint64Key[T ~uint64]() KeyCodec[T] { return uint64Key[T]{} } -type uint64Key[T ~uint64] struct { - name string -} +type uint64Key[T ~uint64] struct{} func (uint64Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint64(buffer, (uint64)(key)) @@ -57,20 +55,9 @@ func (uint64Key[T]) KeyType() string { return "uint64" } -func (u uint64Key[T]) WithName(name string) NamedKeyCodec[T] { - u.name = name - return u -} - -func (u uint64Key[T]) Name() string { - return u.name -} - -func NewUint32Key[T ~uint32]() NameableKeyCodec[T] { return uint32Key[T]{} } +func NewUint32Key[T ~uint32]() KeyCodec[T] { return uint32Key[T]{} } -type uint32Key[T ~uint32] struct { - name string -} +type uint32Key[T ~uint32] struct{} func (uint32Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint32(buffer, (uint32)(key)) @@ -108,20 +95,9 @@ func (u uint32Key[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { return func (uint32Key[T]) SizeNonTerminal(_ T) int { return 4 } -func (u uint32Key[T]) WithName(name string) NamedKeyCodec[T] { - u.name = name - return u -} +func NewUint16Key[T ~uint16]() KeyCodec[T] { return uint16Key[T]{} } -func (u uint32Key[T]) Name() string { - return u.name -} - -func NewUint16Key[T ~uint16]() NameableKeyCodec[T] { return uint16Key[T]{} } - -type uint16Key[T ~uint16] struct { - name string -} +type uint16Key[T ~uint16] struct{} func (uint16Key[T]) Encode(buffer []byte, key T) (int, error) { binary.BigEndian.PutUint16(buffer, (uint16)(key)) @@ -159,15 +135,6 @@ func (u uint16Key[T]) DecodeNonTerminal(buffer []byte) (int, T, error) { return func (u uint16Key[T]) SizeNonTerminal(key T) int { return u.Size(key) } -func (u uint16Key[T]) WithName(name string) NamedKeyCodec[T] { - u.name = name - return u -} - -func (u uint16Key[T]) Name() string { - return u.name -} - func uintEncodeJSON(value uint64) ([]byte, error) { str := `"` + strconv.FormatUint(value, 10) + `"` return []byte(str), nil diff --git a/collections/naming_test.go b/collections/naming_test.go deleted file mode 100644 index 48d05d8d7670..000000000000 --- a/collections/naming_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package collections - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestNaming(t *testing.T) { - require.Equal(t, "u16", Uint16Key.WithName("u16").Name()) - require.Equal(t, "u32", Uint32Key.WithName("u32").Name()) - require.Equal(t, "u64", Uint64Key.WithName("u64").Name()) - require.Equal(t, "i32", Int32Key.WithName("i32").Name()) - require.Equal(t, "i64", Int64Key.WithName("i64").Name()) - require.Equal(t, "str", StringKey.WithName("str").Name()) - require.Equal(t, "bytes", BytesKey.WithName("bytes").Name()) - require.Equal(t, "bool", BoolKey.WithName("bool").Name()) - - require.Equal(t, "vu16", Uint16Value.WithName("vu16").Name()) - require.Equal(t, "vu32", Uint32Value.WithName("vu32").Name()) - require.Equal(t, "vu64", Uint64Value.WithName("vu64").Name()) - require.Equal(t, "vi32", Int32Value.WithName("vi32").Name()) - require.Equal(t, "vi64", Int64Value.WithName("vi64").Name()) - require.Equal(t, "vstr", StringValue.WithName("vstr").Name()) - require.Equal(t, "vbytes", BytesValue.WithName("vbytes").Name()) - require.Equal(t, "vbool", BoolValue.WithName("vbool").Name()) - - require.Equal(t, "abc,def", NamedPairKeyCodec[bool, string]("abc", BoolKey, "def", StringKey).Name()) - require.Equal(t, "abc,def,ghi", NamedTripleKeyCodec[bool, string, int32]("abc", BoolKey, "def", StringKey, "ghi", Int32Key).Name()) -} diff --git a/collections/pair.go b/collections/pair.go index 4f0f91c0d552..f12aaac1b576 100644 --- a/collections/pair.go +++ b/collections/pair.go @@ -6,7 +6,6 @@ import ( "strings" "cosmossdk.io/collections/codec" - "cosmossdk.io/schema" ) // Pair defines a key composed of two keys. @@ -55,22 +54,9 @@ func PairKeyCodec[K1, K2 any](keyCodec1 codec.KeyCodec[K1], keyCodec2 codec.KeyC } } -// NamedPairKeyCodec instantiates a new KeyCodec instance that can encode the Pair, given the KeyCodec of the -// first part of the key and the KeyCodec of the second part of the key, with names assigned to each part -// which will only be used for indexing and informational purposes. -func NamedPairKeyCodec[K1, K2 any](key1Name string, keyCodec1 codec.KeyCodec[K1], key2Name string, keyCodec2 codec.KeyCodec[K2]) codec.NamedKeyCodec[Pair[K1, K2]] { - return pairKeyCodec[K1, K2]{ - key1Name: key1Name, - key2Name: key2Name, - keyCodec1: keyCodec1, - keyCodec2: keyCodec2, - } -} - type pairKeyCodec[K1, K2 any] struct { - key1Name, key2Name string - keyCodec1 codec.KeyCodec[K1] - keyCodec2 codec.KeyCodec[K2] + keyCodec1 codec.KeyCodec[K1] + keyCodec2 codec.KeyCodec[K2] } func (p pairKeyCodec[K1, K2]) KeyCodec1() codec.KeyCodec[K1] { return p.keyCodec1 } @@ -230,69 +216,6 @@ func (p pairKeyCodec[K1, K2]) DecodeJSON(b []byte) (Pair[K1, K2], error) { return Join(k1, k2), nil } -func (p pairKeyCodec[K1, K2]) Name() string { - if p.key1Name == "" || p.key2Name == "" { - return "key1,key2" - } - return fmt.Sprintf("%s,%s", p.key1Name, p.key2Name) -} - -func (p pairKeyCodec[K1, K2]) LogicalDecoder() (res codec.LogicalDecoder[Pair[K1, K2]], err error) { - dec1, err := KeyCodecDecoder(p.keyCodec1) - if err != nil { - return - } - - dec2, err := KeyCodecDecoder(p.keyCodec2) - if err != nil { - return - } - - if len(dec1.Fields) != 1 { - err = fmt.Errorf("key1 codec must have exactly one field") - return - } - if len(dec2.Fields) != 1 { - err = fmt.Errorf("key1 codec must have exactly one field") - return - } - - fields := []schema.Field{dec1.Fields[0], dec2.Fields[0]} - fields[0].Name = p.key1Name - fields[1].Name = p.key2Name - - return codec.LogicalDecoder[Pair[K1, K2]]{ - Fields: fields, - ToSchemaType: func(pair Pair[K1, K2]) (any, error) { - k1, err := dec1.ToSchemaType(*pair.key1) - if err != nil { - return nil, err - } - - k2, err := dec2.ToSchemaType(*pair.key2) - if err != nil { - return nil, err - } - - return []any{k1, k2}, nil - }, - }, nil -} - -//func (p pairKeyCodec[K1, K2]) SchemaColumns() []schema.Field { -// //var k1 K1 -// //col1, _ := extractFields(k1) -// //if len(col1) == 1 { -// // col1[0].Name = p.key1Name -// //} -// //var k2 K2 -// //col2, _ := extractFields(k2) -// //if len(col2) == 1 { -// // col2[0].Name = p.key2Name -// //} -// //return append(col1, col2...) -//} - // NewPrefixUntilPairRange defines a collection query which ranges until the provided Pair prefix. // Unstable: this API might change in the future. func NewPrefixUntilPairRange[K1, K2 any](prefix K1) *PairRange[K1, K2] { diff --git a/collections/triple.go b/collections/triple.go index 25ac7f59f782..9733d9984099 100644 --- a/collections/triple.go +++ b/collections/triple.go @@ -64,22 +64,10 @@ func TripleKeyCodec[K1, K2, K3 any](keyCodec1 codec.KeyCodec[K1], keyCodec2 code } } -func NamedTripleKeyCodec[K1, K2, K3 any](key1Name string, keyCodec1 codec.KeyCodec[K1], key2Name string, keyCodec2 codec.KeyCodec[K2], key3Name string, keyCodec3 codec.KeyCodec[K3]) codec.NamedKeyCodec[Triple[K1, K2, K3]] { - return tripleKeyCodec[K1, K2, K3]{ - key1Name: key1Name, - key2Name: key2Name, - key3Name: key3Name, - keyCodec1: keyCodec1, - keyCodec2: keyCodec2, - keyCodec3: keyCodec3, - } -} - type tripleKeyCodec[K1, K2, K3 any] struct { - key1Name, key2Name, key3Name string - keyCodec1 codec.KeyCodec[K1] - keyCodec2 codec.KeyCodec[K2] - keyCodec3 codec.KeyCodec[K3] + keyCodec1 codec.KeyCodec[K1] + keyCodec2 codec.KeyCodec[K2] + keyCodec3 codec.KeyCodec[K3] } type jsonTripleKey [3]json.RawMessage @@ -285,43 +273,6 @@ func (t tripleKeyCodec[K1, K2, K3]) SizeNonTerminal(key Triple[K1, K2, K3]) int return size } -func (t tripleKeyCodec[K1, K2, K3]) Name() string { - if t.key1Name == "" || t.key2Name == "" || t.key3Name == "" { - return "key1,key2,key3" - } - return fmt.Sprintf("%s,%s,%s", t.key1Name, t.key2Name, t.key3Name) -} - -//func (p tripleKeyCodec[K1, K2, K3]) SchemaColumns() []indexerbase.Column { -// var k1 K1 -// col1, _ := extractFields(k1) -// if len(col1) == 1 { -// col1[0].Name = p.key1Name -// } -// var k2 K2 -// col2, _ := extractFields(k2) -// if len(col2) == 1 { -// col2[0].Name = p.key2Name -// } -// var k3 K3 -// col3, _ := extractFields(k3) -// if len(col3) == 1 { -// col3[0].Name = p.key3Name -// -// } -// cols := append(col1, col2...) -// cols = append(cols, col3...) -// return cols -//} -// -//func (p tripleKeyCodec[K1, K2, K3]) DecodeIndexable(buffer []byte) (any, error) { -// _, x, err := p.Decode(buffer) -// if err != nil { -// return nil, err -// } -// return []any{x.K1(), x.K2(), x.K3()}, nil -//} - // NewPrefixUntilTripleRange defines a collection query which ranges until the provided Pair prefix. // Unstable: this API might change in the future. func NewPrefixUntilTripleRange[K1, K2, K3 any](k1 K1) Ranger[Triple[K1, K2, K3]] { From dcb58dbde4a5e03eb0efbc6adfc926f466bdd545 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 8 Jul 2024 16:28:26 +0200 Subject: [PATCH 47/63] refactoring, docs --- collections/codec/codec.go | 18 ---------------- collections/codec/indexing.go | 39 +++++++++++++++++++++++++++++++++++ collections/indexing.go | 18 ++++++++-------- 3 files changed, 48 insertions(+), 27 deletions(-) create mode 100644 collections/codec/indexing.go diff --git a/collections/codec/codec.go b/collections/codec/codec.go index 83fe732bd55c..2988c9f52425 100644 --- a/collections/codec/codec.go +++ b/collections/codec/codec.go @@ -3,8 +3,6 @@ package codec import ( "errors" "fmt" - - "cosmossdk.io/schema" ) var ErrEncoding = errors.New("collections: encoding error") @@ -76,22 +74,6 @@ type ValueCodec[T any] interface { ValueType() string } -// IndexableCodec is an interface that all codec's should implement in order to properly support indexing. -// It is not required by KeyCodec or ValueCodec in order to preserve backwards compatibility, but -// a future version of collections may make it required and all codec's should aim to implement it. -// If it is not implemented, fallback defaults will be used for indexing that may be sub-optimal. -// -// Implementations of IndexableCodec should test that they are conformant using the schema.ValidateWithKeyFields -// and schema.ValidateWithValueFields depending on whether the codec is a KeyCodec or ValueCodec respectively. -type IndexableCodec[T any] interface { - LogicalDecoder() (LogicalDecoder[T], error) -} - -type LogicalDecoder[T any] struct { - Fields []schema.Field - ToSchemaType func(T) (any, error) -} - // NewUntypedValueCodec returns an UntypedValueCodec for the provided ValueCodec. func NewUntypedValueCodec[V any](v ValueCodec[V]) UntypedValueCodec { typeName := fmt.Sprintf("%T", *new(V)) diff --git a/collections/codec/indexing.go b/collections/codec/indexing.go new file mode 100644 index 000000000000..1cc490f09f81 --- /dev/null +++ b/collections/codec/indexing.go @@ -0,0 +1,39 @@ +package codec + +import "cosmossdk.io/schema" + +// HasSchemaCodec is an interface that all codec's should implement in order +// to properly support indexing. // It is not required by KeyCodec or ValueCodec +// in order to preserve backwards compatibility, but a future version of collections +// may make it required and all codec's should aim to implement it. If it is not +// implemented, fallback defaults will be used for indexing that may be sub-optimal. +// +// Implementations of HasSchemaCodec should test that they are conformant using +// schema.ValidateObjectKey or schema.ValidateObjectValue depending on whether +// the codec is a KeyCodec or ValueCodec respectively. +type HasSchemaCodec[T any] interface { + // SchemaCodec returns the schema codec for the collections codec. + SchemaCodec() (SchemaCodec[T], error) +} + +// SchemaCodec is a codec that supports converting collection codec values to and +// from schema codec values. +type SchemaCodec[T any] struct { + // Fields are the schema fields that the codec represents. If this is empty, + // it will be assumed that this codec represents no value (such as an item key + // or key set value). + Fields []schema.Field + + // ToSchemaType converts a codec value of type T to a value corresponding to + // a schema object key or value (depending on whether this is a key or value + // codec). The returned value should pass validation with schema.ValidateObjectKey + // or schema.ValidateObjectValue with the fields specified in Fields. + // If this function is nil, it will be assumed that T already represents a + // value that conforms to a schema value without any further conversion. + ToSchemaType func(T) (any, error) + + // FromSchemaType converts a schema object key or value to T. + // If this function is nil, it will be assumed that T already represents a + // value that conforms to a schema value without any further conversion. + FromSchemaType func(any) (T, error) +} diff --git a/collections/indexing.go b/collections/indexing.go index e4529359090d..35e99f4b1fc2 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -155,34 +155,34 @@ func (c collectionImpl[K, V]) logicalDecoder() (logicalDecoder, error) { return res, nil } -func KeyCodecDecoder[K any](cdc codec.KeyCodec[K]) (codec.LogicalDecoder[K], error) { - if indexable, ok := cdc.(codec.IndexableCodec[K]); ok { - return indexable.LogicalDecoder() +func KeyCodecDecoder[K any](cdc codec.KeyCodec[K]) (codec.SchemaCodec[K], error) { + if indexable, ok := cdc.(codec.HasSchemaCodec[K]); ok { + return indexable.SchemaCodec() } else { return FallbackDecoder[K](), nil } } -func ValueCodecDecoder[K any](cdc codec.ValueCodec[K]) (codec.LogicalDecoder[K], error) { - if indexable, ok := cdc.(codec.IndexableCodec[K]); ok { - return indexable.LogicalDecoder() +func ValueCodecDecoder[K any](cdc codec.ValueCodec[K]) (codec.SchemaCodec[K], error) { + if indexable, ok := cdc.(codec.HasSchemaCodec[K]); ok { + return indexable.SchemaCodec() } else { return FallbackDecoder[K](), nil } } -func FallbackDecoder[T any]() codec.LogicalDecoder[T] { +func FallbackDecoder[T any]() codec.SchemaCodec[T] { var t T kind := schema.KindForGoValue(t) if err := kind.Validate(); err == nil { - return codec.LogicalDecoder[T]{ + return codec.SchemaCodec[T]{ Fields: []schema.Field{{Kind: kind}}, ToSchemaType: func(t T) (any, error) { return t, nil }, } } else { - return codec.LogicalDecoder[T]{ + return codec.SchemaCodec[T]{ Fields: []schema.Field{{Kind: schema.JSONKind}}, ToSchemaType: func(t T) (any, error) { bz, err := json.Marshal(t) From 2cec5140692ddadbcea045af53026a968e4735a0 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 8 Jul 2024 16:38:09 +0200 Subject: [PATCH 48/63] refactoring --- collections/codec/indexing.go | 58 ++++++++++++++++++++++++++++++++- collections/collections.go | 11 +++++-- collections/indexing.go | 61 +++++++---------------------------- 3 files changed, 77 insertions(+), 53 deletions(-) diff --git a/collections/codec/indexing.go b/collections/codec/indexing.go index 1cc490f09f81..74fd9608d0a5 100644 --- a/collections/codec/indexing.go +++ b/collections/codec/indexing.go @@ -1,6 +1,11 @@ package codec -import "cosmossdk.io/schema" +import ( + "encoding/json" + "fmt" + + "cosmossdk.io/schema" +) // HasSchemaCodec is an interface that all codec's should implement in order // to properly support indexing. // It is not required by KeyCodec or ValueCodec @@ -37,3 +42,54 @@ type SchemaCodec[T any] struct { // value that conforms to a schema value without any further conversion. FromSchemaType func(any) (T, error) } + +func KeySchemaCodec[K any](cdc KeyCodec[K]) (SchemaCodec[K], error) { + if indexable, ok := cdc.(HasSchemaCodec[K]); ok { + return indexable.SchemaCodec() + } else { + return FallbackCodec[K](), nil + } +} + +func ValueSchemaCodec[V any](cdc ValueCodec[V]) (SchemaCodec[V], error) { + if indexable, ok := cdc.(HasSchemaCodec[V]); ok { + return indexable.SchemaCodec() + } else { + return FallbackCodec[V](), nil + } +} + +func FallbackCodec[T any]() SchemaCodec[T] { + var t T + kind := schema.KindForGoValue(t) + if err := kind.Validate(); err == nil { + return SchemaCodec[T]{ + Fields: []schema.Field{{ + // we don't set any name so that this can be set to a good default by the caller + Name: "", + Kind: kind, + }}, + // these can be nil because T maps directly to a schema value for this kind + ToSchemaType: nil, + FromSchemaType: nil, + } + } else { + // we default to encoding everything to JSON + return SchemaCodec[T]{ + Fields: []schema.Field{{Kind: schema.JSONKind}}, + ToSchemaType: func(t T) (any, error) { + bz, err := json.Marshal(t) + return json.RawMessage(bz), err + }, + FromSchemaType: func(a any) (T, error) { + var t T + bz, ok := a.(json.RawMessage) + if !ok { + return t, fmt.Errorf("expected json.RawMessage, got %") + } + err := json.Unmarshal(bz, &t) + return t, err + }, + } + } +} diff --git a/collections/collections.go b/collections/collections.go index 05bfcca2a2be..9879967c50ae 100644 --- a/collections/collections.go +++ b/collections/collections.go @@ -93,15 +93,22 @@ type Collection interface { genesisHandler - logicalDecoder() (logicalDecoder, error) + // schemaCodec returns the schema codec for this collection. + schemaCodec() (schemaCodec, error) + // isSecondaryIndex indicates that this collection represents a secondary index + // in the schema and should be excluded from the module's user facing schema. isSecondaryIndex() bool } -type logicalDecoder struct { +// schemaCodec maps a collection to a schema object type and provides +// decoders and encoders to and from schema values and raw kv-store bytes. +type schemaCodec struct { objectType schema.ObjectType keyDecoder func([]byte) (any, error) valueDecoder func([]byte) (any, error) + keyEncoder func(any) ([]byte, error) + valueEncoder func(any) ([]byte, error) } // Prefix defines a segregation bytes namespace for specific collections objects. diff --git a/collections/indexing.go b/collections/indexing.go index 35e99f4b1fc2..2620b4b2c950 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -2,15 +2,13 @@ package collections import ( "bytes" - "encoding/json" "fmt" "strings" "github.com/tidwall/btree" - "cosmossdk.io/schema" - "cosmossdk.io/collections/codec" + "cosmossdk.io/schema" ) type IndexingOptions struct { @@ -37,7 +35,7 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { continue } - ld, err := coll.logicalDecoder() + ld, err := coll.schemaCodec() if err != nil { return schema.ModuleCodec{}, err } @@ -48,8 +46,8 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { objectTypes = append(objectTypes, ld.objectType) decoder.collectionLookup.Set(string(coll.GetPrefix()), &collDecoder{ - Collection: coll, - logicalDecoder: ld, + Collection: coll, + schemaCodec: ld, }) } @@ -86,7 +84,7 @@ func (m moduleDecoder) decodeKV(update schema.KVPairUpdate) ([]schema.ObjectUpda type collDecoder struct { Collection - logicalDecoder + schemaCodec } func (c collDecoder) decodeKVPair(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { @@ -120,13 +118,13 @@ func (c collDecoder) decodeKVPair(update schema.KVPairUpdate) ([]schema.ObjectUp }, nil } -func (c collectionImpl[K, V]) logicalDecoder() (logicalDecoder, error) { - res := logicalDecoder{} +func (c collectionImpl[K, V]) schemaCodec() (schemaCodec, error) { + res := schemaCodec{} res.objectType.Name = c.GetName() - keyDecoder, err := KeyCodecDecoder(c.m.kc) + keyDecoder, err := codec.KeySchemaCodec(c.m.kc) if err != nil { - return logicalDecoder{}, err + return schemaCodec{}, err } res.objectType.KeyFields = keyDecoder.Fields res.keyDecoder = func(i []byte) (any, error) { @@ -138,9 +136,9 @@ func (c collectionImpl[K, V]) logicalDecoder() (logicalDecoder, error) { } ensureFieldNames(c.m.kc, "key", res.objectType.KeyFields) - valueDecoder, err := ValueCodecDecoder(c.m.vc) + valueDecoder, err := codec.ValueSchemaCodec(c.m.vc) if err != nil { - return logicalDecoder{}, err + return schemaCodec{}, err } res.objectType.ValueFields = valueDecoder.Fields res.valueDecoder = func(i []byte) (any, error) { @@ -155,43 +153,6 @@ func (c collectionImpl[K, V]) logicalDecoder() (logicalDecoder, error) { return res, nil } -func KeyCodecDecoder[K any](cdc codec.KeyCodec[K]) (codec.SchemaCodec[K], error) { - if indexable, ok := cdc.(codec.HasSchemaCodec[K]); ok { - return indexable.SchemaCodec() - } else { - return FallbackDecoder[K](), nil - } -} - -func ValueCodecDecoder[K any](cdc codec.ValueCodec[K]) (codec.SchemaCodec[K], error) { - if indexable, ok := cdc.(codec.HasSchemaCodec[K]); ok { - return indexable.SchemaCodec() - } else { - return FallbackDecoder[K](), nil - } -} - -func FallbackDecoder[T any]() codec.SchemaCodec[T] { - var t T - kind := schema.KindForGoValue(t) - if err := kind.Validate(); err == nil { - return codec.SchemaCodec[T]{ - Fields: []schema.Field{{Kind: kind}}, - ToSchemaType: func(t T) (any, error) { - return t, nil - }, - } - } else { - return codec.SchemaCodec[T]{ - Fields: []schema.Field{{Kind: schema.JSONKind}}, - ToSchemaType: func(t T) (any, error) { - bz, err := json.Marshal(t) - return json.RawMessage(bz), err - }, - } - } -} - func ensureFieldNames(x any, defaultName string, cols []schema.Field) { var names []string = nil if hasName, ok := x.(interface{ Name() string }); ok { From cd9be914fe052f262e0525f310547419e19c927a Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 8 Jul 2024 16:43:12 +0200 Subject: [PATCH 49/63] refactoring --- collections/indexing.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/collections/indexing.go b/collections/indexing.go index 2620b4b2c950..27fea21807cf 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -11,11 +11,14 @@ import ( "cosmossdk.io/schema" ) +// IndexingOptions are indexing options for the collections schema. type IndexingOptions struct { + // RetainDeletionsFor is the list of collections to retain deletions for. RetainDeletionsFor []string } +// ModuleCodec returns the ModuleCodec for this schema for the provided options. func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { decoder := moduleDecoder{ collectionLookup: &btree.Map[string, *collDecoder]{}, @@ -40,11 +43,12 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { return schema.ModuleCodec{}, err } - if !retainDeletions[coll.GetName()] { + if retainDeletions[coll.GetName()] { ld.objectType.RetainDeletions = true } objectTypes = append(objectTypes, ld.objectType) + decoder.collectionLookup.Set(string(coll.GetPrefix()), &collDecoder{ Collection: coll, schemaCodec: ld, @@ -60,6 +64,7 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { } type moduleDecoder struct { + // collectionLookup lets us efficiently look the correct collection based on raw key bytes collectionLookup *btree.Map[string, *collDecoder] } @@ -67,6 +72,7 @@ func (m moduleDecoder) decodeKV(update schema.KVPairUpdate) ([]schema.ObjectUpda key := update.Key ks := string(key) var cd *collDecoder + // we look for the collection whose prefix is less than this key m.collectionLookup.Descend(ks, func(prefix string, cur *collDecoder) bool { bytesPrefix := cur.GetPrefix() if bytes.HasPrefix(key, bytesPrefix) { @@ -153,6 +159,8 @@ func (c collectionImpl[K, V]) schemaCodec() (schemaCodec, error) { return res, nil } +// ensureFieldNames makes sure that all fields have valid names - either the +// names were specified by user or they get filled in with defaults here func ensureFieldNames(x any, defaultName string, cols []schema.Field) { var names []string = nil if hasName, ok := x.(interface{ Name() string }); ok { From feab7a14dc98c15572fba713b7e2454420cdde23 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 8 Jul 2024 16:49:20 +0200 Subject: [PATCH 50/63] refactoring --- collections/codec/indexing.go | 2 +- collections/collections.go | 9 ++++--- collections/indexing.go | 50 +++++++++++++++-------------------- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/collections/codec/indexing.go b/collections/codec/indexing.go index 74fd9608d0a5..e7de487e9eb2 100644 --- a/collections/codec/indexing.go +++ b/collections/codec/indexing.go @@ -85,7 +85,7 @@ func FallbackCodec[T any]() SchemaCodec[T] { var t T bz, ok := a.(json.RawMessage) if !ok { - return t, fmt.Errorf("expected json.RawMessage, got %") + return t, fmt.Errorf("expected json.RawMessage, got %T", a) } err := json.Unmarshal(bz, &t) return t, err diff --git a/collections/collections.go b/collections/collections.go index 9879967c50ae..24eca492fc91 100644 --- a/collections/collections.go +++ b/collections/collections.go @@ -93,17 +93,18 @@ type Collection interface { genesisHandler - // schemaCodec returns the schema codec for this collection. - schemaCodec() (schemaCodec, error) + // collectionSchemaCodec returns the schema codec for this collection. + schemaCodec() (*collectionSchemaCodec, error) // isSecondaryIndex indicates that this collection represents a secondary index // in the schema and should be excluded from the module's user facing schema. isSecondaryIndex() bool } -// schemaCodec maps a collection to a schema object type and provides +// collectionSchemaCodec maps a collection to a schema object type and provides // decoders and encoders to and from schema values and raw kv-store bytes. -type schemaCodec struct { +type collectionSchemaCodec struct { + coll Collection objectType schema.ObjectType keyDecoder func([]byte) (any, error) valueDecoder func([]byte) (any, error) diff --git a/collections/indexing.go b/collections/indexing.go index 27fea21807cf..b0cf65c03d99 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -21,7 +21,7 @@ type IndexingOptions struct { // ModuleCodec returns the ModuleCodec for this schema for the provided options. func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { decoder := moduleDecoder{ - collectionLookup: &btree.Map[string, *collDecoder]{}, + collectionLookup: &btree.Map[string, *collectionSchemaCodec]{}, } retainDeletions := make(map[string]bool) @@ -38,21 +38,18 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { continue } - ld, err := coll.schemaCodec() + cdc, err := coll.schemaCodec() if err != nil { return schema.ModuleCodec{}, err } if retainDeletions[coll.GetName()] { - ld.objectType.RetainDeletions = true + cdc.objectType.RetainDeletions = true } - objectTypes = append(objectTypes, ld.objectType) + objectTypes = append(objectTypes, cdc.objectType) - decoder.collectionLookup.Set(string(coll.GetPrefix()), &collDecoder{ - Collection: coll, - schemaCodec: ld, - }) + decoder.collectionLookup.Set(string(coll.GetPrefix()), cdc) } return schema.ModuleCodec{ @@ -65,16 +62,16 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { type moduleDecoder struct { // collectionLookup lets us efficiently look the correct collection based on raw key bytes - collectionLookup *btree.Map[string, *collDecoder] + collectionLookup *btree.Map[string, *collectionSchemaCodec] } func (m moduleDecoder) decodeKV(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { key := update.Key ks := string(key) - var cd *collDecoder + var cd *collectionSchemaCodec // we look for the collection whose prefix is less than this key - m.collectionLookup.Descend(ks, func(prefix string, cur *collDecoder) bool { - bytesPrefix := cur.GetPrefix() + m.collectionLookup.Descend(ks, func(prefix string, cur *collectionSchemaCodec) bool { + bytesPrefix := cur.coll.GetPrefix() if bytes.HasPrefix(key, bytesPrefix) { cd = cur return true @@ -88,49 +85,46 @@ func (m moduleDecoder) decodeKV(update schema.KVPairUpdate) ([]schema.ObjectUpda return cd.decodeKVPair(update) } -type collDecoder struct { - Collection - schemaCodec -} - -func (c collDecoder) decodeKVPair(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { +func (c collectionSchemaCodec) decodeKVPair(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { // strip prefix key := update.Key - key = key[len(c.GetPrefix()):] + key = key[len(c.coll.GetPrefix()):] k, err := c.keyDecoder(key) if err != nil { return []schema.ObjectUpdate{ - {TypeName: c.GetName()}, + {TypeName: c.coll.GetName()}, }, err } if update.Delete { return []schema.ObjectUpdate{ - {TypeName: c.GetName(), Key: k, Delete: true}, + {TypeName: c.coll.GetName(), Key: k, Delete: true}, }, nil } v, err := c.valueDecoder(update.Value) if err != nil { return []schema.ObjectUpdate{ - {TypeName: c.GetName(), Key: k}, + {TypeName: c.coll.GetName(), Key: k}, }, err } return []schema.ObjectUpdate{ - {TypeName: c.GetName(), Key: k, Value: v}, + {TypeName: c.coll.GetName(), Key: k, Value: v}, }, nil } -func (c collectionImpl[K, V]) schemaCodec() (schemaCodec, error) { - res := schemaCodec{} +func (c collectionImpl[K, V]) schemaCodec() (*collectionSchemaCodec, error) { + res := &collectionSchemaCodec{ + coll: c, + } res.objectType.Name = c.GetName() keyDecoder, err := codec.KeySchemaCodec(c.m.kc) if err != nil { - return schemaCodec{}, err + return nil, err } res.objectType.KeyFields = keyDecoder.Fields res.keyDecoder = func(i []byte) (any, error) { @@ -144,7 +138,7 @@ func (c collectionImpl[K, V]) schemaCodec() (schemaCodec, error) { valueDecoder, err := codec.ValueSchemaCodec(c.m.vc) if err != nil { - return schemaCodec{}, err + return nil, err } res.objectType.ValueFields = valueDecoder.Fields res.valueDecoder = func(i []byte) (any, error) { @@ -160,7 +154,7 @@ func (c collectionImpl[K, V]) schemaCodec() (schemaCodec, error) { } // ensureFieldNames makes sure that all fields have valid names - either the -// names were specified by user or they get filled in with defaults here +// names were specified by user or they get filleed func ensureFieldNames(x any, defaultName string, cols []schema.Field) { var names []string = nil if hasName, ok := x.(interface{ Name() string }); ok { From d87834918f781afa0b26b1479c3295dd52ebb1ef Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 8 Jul 2024 16:50:31 +0200 Subject: [PATCH 51/63] docs --- collections/map.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/collections/map.go b/collections/map.go index dec5bd19ebfa..360d96feafa3 100644 --- a/collections/map.go +++ b/collections/map.go @@ -17,9 +17,13 @@ type Map[K, V any] struct { vc codec.ValueCodec[V] // store accessor - sa func(context.Context) store.KVStore - prefix []byte - name string + sa func(context.Context) store.KVStore + prefix []byte + name string + + // isSecondaryIndex indicates that this map represents a secondary index + // on another collection and that it should be skipped when generating + // a user facing schema isSecondaryIndex bool } From 826528a511e971cb3085664cbd8ff9affbf39b34 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 8 Jul 2024 16:52:36 +0200 Subject: [PATCH 52/63] docs --- collections/codec/indexing.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/collections/codec/indexing.go b/collections/codec/indexing.go index e7de487e9eb2..95450dd463cf 100644 --- a/collections/codec/indexing.go +++ b/collections/codec/indexing.go @@ -43,23 +43,30 @@ type SchemaCodec[T any] struct { FromSchemaType func(any) (T, error) } +// KeySchemaCodec gets the schema codec for the provided KeyCodec either +// by casting to HasSchemaCodec or returning a fallback codec. func KeySchemaCodec[K any](cdc KeyCodec[K]) (SchemaCodec[K], error) { if indexable, ok := cdc.(HasSchemaCodec[K]); ok { return indexable.SchemaCodec() } else { - return FallbackCodec[K](), nil + return FallbackSchemaCodec[K](), nil } } +// ValueSchemaCodec gets the schema codec for the provided ValueCodec either +// by casting to HasSchemaCodec or returning a fallback codec. func ValueSchemaCodec[V any](cdc ValueCodec[V]) (SchemaCodec[V], error) { if indexable, ok := cdc.(HasSchemaCodec[V]); ok { return indexable.SchemaCodec() } else { - return FallbackCodec[V](), nil + return FallbackSchemaCodec[V](), nil } } -func FallbackCodec[T any]() SchemaCodec[T] { +// FallbackSchemaCodec returns a fallback schema codec for T when one isn't explicitly +// specified with HasSchemaCodec. It maps all simple types directly to schema kinds +// and converts everything else to JSON. +func FallbackSchemaCodec[T any]() SchemaCodec[T] { var t T kind := schema.KindForGoValue(t) if err := kind.Validate(); err == nil { From a39f8b3e30c566f32be9ee65f521dd45c018d652 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 9 Jul 2024 12:17:46 +0200 Subject: [PATCH 53/63] remove go.mod replace --- collections/go.mod | 3 +-- collections/go.sum | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/collections/go.mod b/collections/go.mod index 1cbf81c30423..6a3d475995ec 100644 --- a/collections/go.mod +++ b/collections/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( cosmossdk.io/core v0.12.0 cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 - cosmossdk.io/schema v0.0.0 + cosmossdk.io/schema v0.1.0 github.com/stretchr/testify v1.9.0 github.com/tidwall/btree v1.7.0 pgregory.net/rapid v1.1.0 @@ -31,5 +31,4 @@ require ( replace ( cosmossdk.io/core => ../core cosmossdk.io/core/testing => ../core/testing - cosmossdk.io/schema => ../schema ) diff --git a/collections/go.sum b/collections/go.sum index 009288c70d59..eb19745545fa 100644 --- a/collections/go.sum +++ b/collections/go.sum @@ -1,3 +1,5 @@ +cosmossdk.io/schema v0.1.0 h1:HZz2kmC+o/Xjsd5BrQHT3cZU5F450l2uGoDvmpIYI9s= +cosmossdk.io/schema v0.1.0/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= github.com/cosmos/gogoproto v1.5.0 h1:SDVwzEqZDDBoslaeZg+dGE55hdzHfgUA40pEanMh52o= github.com/cosmos/gogoproto v1.5.0/go.mod h1:iUM31aofn3ymidYG6bUR5ZFrk+Om8p5s754eMUcyp8I= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= From e8c46073fd69ba32c1fd631d90dc819a30a96af7 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 6 Aug 2024 10:31:41 -0400 Subject: [PATCH 54/63] tidy --- collections/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/collections/go.mod b/collections/go.mod index a6867432f9f0..d80825eb6c62 100644 --- a/collections/go.mod +++ b/collections/go.mod @@ -16,8 +16,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/tidwall/btree v1.7.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sys v0.22.0 // indirect From f6801cacfd637bea4a0fc862a21a2b749a3b50dd Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 6 Aug 2024 10:57:13 -0400 Subject: [PATCH 55/63] CHANGELOG.md --- collections/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/collections/CHANGELOG.md b/collections/CHANGELOG.md index fda6ca900b30..dbcd8989870f 100644 --- a/collections/CHANGELOG.md +++ b/collections/CHANGELOG.md @@ -37,6 +37,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#18933](https://github.com/cosmos/cosmos-sdk/pull/18933) Add LookupMap implementation. It is basic wrapping of the standard Map methods but is not iterable. * [#17656](https://github.com/cosmos/cosmos-sdk/pull/17656) Introduces `Vec`, a collection type that allows to represent a growable array on top of a KVStore. * [#19861](https://github.com/cosmos/cosmos-sdk/pull/19861) Add `NewJSONValueCodec` value codec as an alternative for `codec.CollValue` from the SDK for non protobuf types. +* [#20704](https://github.com/cosmos/cosmos-sdk/pull/20704) Add `ModuleCodec` method to `Schema` and `HasSchemaCodec` interface in order to support `cosmossdk.io/schema` compatible indexing. ## [v0.4.0](https://github.com/cosmos/cosmos-sdk/releases/tag/collections%2Fv0.4.0) From 69fba76d4e9110ea0fc80dbfeb229a2609ceee42 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 6 Aug 2024 12:03:46 -0400 Subject: [PATCH 56/63] tidy --- x/accounts/defaults/lockup/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/x/accounts/defaults/lockup/go.mod b/x/accounts/defaults/lockup/go.mod index dcf9fadd0381..deb7d20d127a 100644 --- a/x/accounts/defaults/lockup/go.mod +++ b/x/accounts/defaults/lockup/go.mod @@ -19,6 +19,7 @@ require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/depinject v1.0.0 // indirect + cosmossdk.io/schema v0.1.1 // indirect github.com/cometbft/cometbft/api v1.0.0-rc.1 // indirect github.com/cosmos/crypto v0.1.2 // indirect github.com/dgraph-io/badger/v4 v4.2.0 // indirect From 333415f902055dff4235118cd9f2e1223806e557 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 19 Aug 2024 11:22:46 -0400 Subject: [PATCH 57/63] tidy --- collections/go.mod | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/collections/go.mod b/collections/go.mod index 648d548490c6..b7c2027a6147 100644 --- a/collections/go.mod +++ b/collections/go.mod @@ -14,18 +14,11 @@ require ( require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5 // indirect - google.golang.org/grpc v1.64.1 // indirect - google.golang.org/protobuf v1.34.2 // indirect - github.com/tidwall/btree v1.7.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( cosmossdk.io/core => ../core cosmossdk.io/core/testing => ../core/testing + cosmossdk.io/schema => ../schema ) From 1a3429ec6bf33cdf215d23d6d43d22a8efb5ff24 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 20 Aug 2024 09:24:16 -0400 Subject: [PATCH 58/63] Update collections/codec/indexing.go Co-authored-by: Facundo Medica <14063057+facundomedica@users.noreply.github.com> --- collections/codec/indexing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collections/codec/indexing.go b/collections/codec/indexing.go index 95450dd463cf..26ff651cc8e0 100644 --- a/collections/codec/indexing.go +++ b/collections/codec/indexing.go @@ -8,7 +8,7 @@ import ( ) // HasSchemaCodec is an interface that all codec's should implement in order -// to properly support indexing. // It is not required by KeyCodec or ValueCodec +// to properly support indexing. It is not required by KeyCodec or ValueCodec // in order to preserve backwards compatibility, but a future version of collections // may make it required and all codec's should aim to implement it. If it is not // implemented, fallback defaults will be used for indexing that may be sub-optimal. From 57916292b5b65bf8b6448b538530089b9b7807d8 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 20 Aug 2024 09:24:24 -0400 Subject: [PATCH 59/63] Update collections/indexing.go Co-authored-by: Facundo Medica <14063057+facundomedica@users.noreply.github.com> --- collections/indexing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collections/indexing.go b/collections/indexing.go index b0cf65c03d99..88db4b114b3a 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -154,7 +154,7 @@ func (c collectionImpl[K, V]) schemaCodec() (*collectionSchemaCodec, error) { } // ensureFieldNames makes sure that all fields have valid names - either the -// names were specified by user or they get filleed +// names were specified by user or they get filled func ensureFieldNames(x any, defaultName string, cols []schema.Field) { var names []string = nil if hasName, ok := x.(interface{ Name() string }); ok { From b2c0fe28b477a4bbcce376efe74981653630e4b1 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 28 Aug 2024 13:36:51 -0400 Subject: [PATCH 60/63] go mod tidy --- x/accounts/defaults/lockup/go.mod | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x/accounts/defaults/lockup/go.mod b/x/accounts/defaults/lockup/go.mod index 65c15267c762..1065591049fc 100644 --- a/x/accounts/defaults/lockup/go.mod +++ b/x/accounts/defaults/lockup/go.mod @@ -13,7 +13,10 @@ require ( github.com/cosmos/gogoproto v1.7.0 ) -require github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect +require ( + cosmossdk.io/schema v0.1.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect +) require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect From c4f925133270383fce77f0bfec2970e5d1d81a83 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 28 Aug 2024 13:39:40 -0400 Subject: [PATCH 61/63] update to schema main --- collections/indexing.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/collections/indexing.go b/collections/indexing.go index 88db4b114b3a..89527bf8d8f5 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -29,7 +29,7 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { retainDeletions[collName] = true } - var objectTypes []schema.ObjectType + var types []schema.Type for _, collName := range s.collectionsOrdered { coll := s.collectionsByName[collName] @@ -47,15 +47,18 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { cdc.objectType.RetainDeletions = true } - objectTypes = append(objectTypes, cdc.objectType) + types = append(types, cdc.objectType) decoder.collectionLookup.Set(string(coll.GetPrefix()), cdc) } + modSchema, err := schema.NewModuleSchema(types...) + if err != nil { + return schema.ModuleCodec{}, err + } + return schema.ModuleCodec{ - Schema: schema.ModuleSchema{ - ObjectTypes: objectTypes, - }, + Schema: modSchema, KVDecoder: decoder.decodeKV, }, nil } @@ -98,7 +101,7 @@ func (c collectionSchemaCodec) decodeKVPair(update schema.KVPairUpdate) ([]schem } - if update.Delete { + if update.Remove { return []schema.ObjectUpdate{ {TypeName: c.coll.GetName(), Key: k, Delete: true}, }, nil From 2c9d4cf878268023efaabe266f058e73289ae832 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 30 Aug 2024 17:03:23 -0400 Subject: [PATCH 62/63] go mod tidy --- collections/go.mod | 3 +-- collections/go.sum | 2 ++ x/accounts/defaults/lockup/go.mod | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/collections/go.mod b/collections/go.mod index b7c2027a6147..3eaa69315d3c 100644 --- a/collections/go.mod +++ b/collections/go.mod @@ -5,7 +5,7 @@ go 1.23 require ( cosmossdk.io/core v1.0.0 cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 - cosmossdk.io/schema v0.1.0 + cosmossdk.io/schema v0.2.0 github.com/stretchr/testify v1.9.0 github.com/tidwall/btree v1.7.0 pgregory.net/rapid v1.1.0 @@ -20,5 +20,4 @@ require ( replace ( cosmossdk.io/core => ../core cosmossdk.io/core/testing => ../core/testing - cosmossdk.io/schema => ../schema ) diff --git a/collections/go.sum b/collections/go.sum index f59be83bdbe3..a616ad38d9a9 100644 --- a/collections/go.sum +++ b/collections/go.sum @@ -1,3 +1,5 @@ +cosmossdk.io/schema v0.2.0 h1:UH5CR1DqUq8yP+5Np8PbvG4YX0zAUsTN2Qk6yThmfMk= +cosmossdk.io/schema v0.2.0/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/x/accounts/defaults/lockup/go.mod b/x/accounts/defaults/lockup/go.mod index 9e268effbe94..70b590b78de9 100644 --- a/x/accounts/defaults/lockup/go.mod +++ b/x/accounts/defaults/lockup/go.mod @@ -14,7 +14,7 @@ require ( ) require ( - cosmossdk.io/schema v0.1.1 // indirect + cosmossdk.io/schema v0.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect ) From b1610ac8a05e85c38debdc6e365cfe965e0a41c0 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 30 Aug 2024 17:06:03 -0400 Subject: [PATCH 63/63] update to schema 0.2.0 --- collections/indexing.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collections/indexing.go b/collections/indexing.go index 89527bf8d8f5..6fbf5aa0488e 100644 --- a/collections/indexing.go +++ b/collections/indexing.go @@ -5,10 +5,10 @@ import ( "fmt" "strings" + "cosmossdk.io/schema" "github.com/tidwall/btree" "cosmossdk.io/collections/codec" - "cosmossdk.io/schema" ) // IndexingOptions are indexing options for the collections schema. @@ -52,7 +52,7 @@ func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { decoder.collectionLookup.Set(string(coll.GetPrefix()), cdc) } - modSchema, err := schema.NewModuleSchema(types...) + modSchema, err := schema.CompileModuleSchema(types...) if err != nil { return schema.ModuleCodec{}, err }