feat(collections): support indexing #20704
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| package codec | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
|
|
||
| "cosmossdk.io/schema" | ||
| ) | ||
|
|
||
| // HasSchemaCodec is an interface that all codec's should implement in order | ||
| // to properly support indexing. It is not required by KeyCodec or ValueCodec | ||
| // in order to preserve backwards compatibility, but a future version of collections | ||
| // may make it required and all codec's should aim to implement it. If it is not | ||
| // implemented, fallback defaults will be used for indexing that may be sub-optimal. | ||
| // | ||
| // Implementations of HasSchemaCodec should test that they are conformant using | ||
| // schema.ValidateObjectKey or schema.ValidateObjectValue depending on whether | ||
| // the codec is a KeyCodec or ValueCodec respectively. | ||
| type HasSchemaCodec[T any] interface { | ||
| // SchemaCodec returns the schema codec for the collections codec. | ||
| SchemaCodec() (SchemaCodec[T], error) | ||
| } | ||
|
|
||
| // SchemaCodec is a codec that supports converting collection codec values to and | ||
| // from schema codec values. | ||
| type SchemaCodec[T any] struct { | ||
| // Fields are the schema fields that the codec represents. If this is empty, | ||
| // it will be assumed that this codec represents no value (such as an item key | ||
| // or key set value). | ||
| Fields []schema.Field | ||
|
|
||
| // ToSchemaType converts a codec value of type T to a value corresponding to | ||
| // a schema object key or value (depending on whether this is a key or value | ||
| // codec). The returned value should pass validation with schema.ValidateObjectKey | ||
| // or schema.ValidateObjectValue with the fields specified in Fields. | ||
| // If this function is nil, it will be assumed that T already represents a | ||
| // value that conforms to a schema value without any further conversion. | ||
| ToSchemaType func(T) (any, error) | ||
|
|
||
| // FromSchemaType converts a schema object key or value to T. | ||
| // If this function is nil, it will be assumed that T already represents a | ||
| // value that conforms to a schema value without any further conversion. | ||
| FromSchemaType func(any) (T, error) | ||
| } | ||
|
|
||
| // KeySchemaCodec gets the schema codec for the provided KeyCodec either | ||
| // by casting to HasSchemaCodec or returning a fallback codec. | ||
| func KeySchemaCodec[K any](cdc KeyCodec[K]) (SchemaCodec[K], error) { | ||
| if indexable, ok := cdc.(HasSchemaCodec[K]); ok { | ||
| return indexable.SchemaCodec() | ||
| } else { | ||
| return FallbackSchemaCodec[K](), nil | ||
| } | ||
| } | ||
|
|
||
| // ValueSchemaCodec gets the schema codec for the provided ValueCodec either | ||
| // by casting to HasSchemaCodec or returning a fallback codec. | ||
| func ValueSchemaCodec[V any](cdc ValueCodec[V]) (SchemaCodec[V], error) { | ||
| if indexable, ok := cdc.(HasSchemaCodec[V]); ok { | ||
| return indexable.SchemaCodec() | ||
| } else { | ||
| return FallbackSchemaCodec[V](), nil | ||
| } | ||
| } | ||
|
|
||
| // FallbackSchemaCodec returns a fallback schema codec for T when one isn't explicitly | ||
| // specified with HasSchemaCodec. It maps all simple types directly to schema kinds | ||
| // and converts everything else to JSON. | ||
| func FallbackSchemaCodec[T any]() SchemaCodec[T] { | ||
| var t T | ||
| kind := schema.KindForGoValue(t) | ||
| if err := kind.Validate(); err == nil { | ||
| return SchemaCodec[T]{ | ||
| Fields: []schema.Field{{ | ||
| // we don't set any name so that this can be set to a good default by the caller | ||
| Name: "", | ||
| Kind: kind, | ||
| }}, | ||
| // these can be nil because T maps directly to a schema value for this kind | ||
| ToSchemaType: nil, | ||
| FromSchemaType: nil, | ||
| } | ||
| } else { | ||
| // we default to encoding everything to JSON | ||
| return SchemaCodec[T]{ | ||
| Fields: []schema.Field{{Kind: schema.JSONKind}}, | ||
| ToSchemaType: func(t T) (any, error) { | ||
| bz, err := json.Marshal(t) | ||
| return json.RawMessage(bz), err | ||
| }, | ||
| FromSchemaType: func(a any) (T, error) { | ||
| var t T | ||
| bz, ok := a.(json.RawMessage) | ||
| if !ok { | ||
| return t, fmt.Errorf("expected json.RawMessage, got %T", a) | ||
| } | ||
| err := json.Unmarshal(bz, &t) | ||
| return t, err | ||
| }, | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| package collections | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "cosmossdk.io/schema" | ||
| "github.com/tidwall/btree" | ||
|
|
||
| "cosmossdk.io/collections/codec" | ||
| ) | ||
|
|
||
| // IndexingOptions are indexing options for the collections schema. | ||
| type IndexingOptions struct { | ||
|
|
||
| // RetainDeletionsFor is the list of collections to retain deletions for. | ||
| RetainDeletionsFor []string | ||
| } | ||
|
|
||
| // ModuleCodec returns the ModuleCodec for this schema for the provided options. | ||
| func (s Schema) ModuleCodec(opts IndexingOptions) (schema.ModuleCodec, error) { | ||
| decoder := moduleDecoder{ | ||
| collectionLookup: &btree.Map[string, *collectionSchemaCodec]{}, | ||
| } | ||
|
|
||
| retainDeletions := make(map[string]bool) | ||
| for _, collName := range opts.RetainDeletionsFor { | ||
| retainDeletions[collName] = true | ||
| } | ||
|
|
||
| var types []schema.Type | ||
| for _, collName := range s.collectionsOrdered { | ||
| coll := s.collectionsByName[collName] | ||
|
|
||
| // skip secondary indexes | ||
| if coll.isSecondaryIndex() { | ||
| continue | ||
| } | ||
|
|
||
| cdc, err := coll.schemaCodec() | ||
| if err != nil { | ||
| return schema.ModuleCodec{}, err | ||
| } | ||
|
|
||
| if retainDeletions[coll.GetName()] { | ||
| cdc.objectType.RetainDeletions = true | ||
| } | ||
|
|
||
| types = append(types, cdc.objectType) | ||
|
|
||
| decoder.collectionLookup.Set(string(coll.GetPrefix()), cdc) | ||
| } | ||
|
|
||
| modSchema, err := schema.CompileModuleSchema(types...) | ||
| if err != nil { | ||
| return schema.ModuleCodec{}, err | ||
| } | ||
|
|
||
| return schema.ModuleCodec{ | ||
| Schema: modSchema, | ||
| KVDecoder: decoder.decodeKV, | ||
| }, nil | ||
| } | ||
|
|
||
| type moduleDecoder struct { | ||
| // collectionLookup lets us efficiently look the correct collection based on raw key bytes | ||
| collectionLookup *btree.Map[string, *collectionSchemaCodec] | ||
| } | ||
|
|
||
| func (m moduleDecoder) decodeKV(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { | ||
| key := update.Key | ||
| ks := string(key) | ||
| var cd *collectionSchemaCodec | ||
| // we look for the collection whose prefix is less than this key | ||
| m.collectionLookup.Descend(ks, func(prefix string, cur *collectionSchemaCodec) bool { | ||
| bytesPrefix := cur.coll.GetPrefix() | ||
| if bytes.HasPrefix(key, bytesPrefix) { | ||
| cd = cur | ||
| return true | ||
| } | ||
| return false | ||
| }) | ||
| if cd == nil { | ||
| return nil, nil | ||
| } | ||
|
|
||
| return cd.decodeKVPair(update) | ||
| } | ||
|
|
||
| func (c collectionSchemaCodec) decodeKVPair(update schema.KVPairUpdate) ([]schema.ObjectUpdate, error) { | ||
| // strip prefix | ||
| key := update.Key | ||
| key = key[len(c.coll.GetPrefix()):] | ||
|
|
||
| k, err := c.keyDecoder(key) | ||
| if err != nil { | ||
| return []schema.ObjectUpdate{ | ||
| {TypeName: c.coll.GetName()}, | ||
| }, err | ||
|
|
||
| } | ||
|
|
||
| if update.Remove { | ||
| return []schema.ObjectUpdate{ | ||
| {TypeName: c.coll.GetName(), Key: k, Delete: true}, | ||
| }, nil | ||
| } | ||
|
|
||
| v, err := c.valueDecoder(update.Value) | ||
| if err != nil { | ||
| return []schema.ObjectUpdate{ | ||
| {TypeName: c.coll.GetName(), Key: k}, | ||
| }, err | ||
| } | ||
|
|
||
| return []schema.ObjectUpdate{ | ||
| {TypeName: c.coll.GetName(), Key: k, Value: v}, | ||
| }, nil | ||
| } | ||
|
Comment on lines
+91
to
+120
There was a problem hiding this comment. Address undefined field. The function is well-structured and handles key-value pair updates correctly. However, the field Ensure this field is correctly defined or imported. ToolsGitHub Check: tests (03)
GitHub Check: tests (02)
GitHub Check: tests (01)
GitHub Check: tests (00)
GitHub Check: golangci-lint
GitHub Check: dependency-review
|
||
|
|
||
| func (c collectionImpl[K, V]) schemaCodec() (*collectionSchemaCodec, error) { | ||
| res := &collectionSchemaCodec{ | ||
| coll: c, | ||
| } | ||
| res.objectType.Name = c.GetName() | ||
|
|
||
| keyDecoder, err := codec.KeySchemaCodec(c.m.kc) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| res.objectType.KeyFields = keyDecoder.Fields | ||
| res.keyDecoder = func(i []byte) (any, error) { | ||
| _, x, err := c.m.kc.Decode(i) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return keyDecoder.ToSchemaType(x) | ||
| } | ||
| ensureFieldNames(c.m.kc, "key", res.objectType.KeyFields) | ||
|
|
||
| valueDecoder, err := codec.ValueSchemaCodec(c.m.vc) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| res.objectType.ValueFields = valueDecoder.Fields | ||
| res.valueDecoder = func(i []byte) (any, error) { | ||
| x, err := c.m.vc.Decode(i) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return valueDecoder.ToSchemaType(x) | ||
| } | ||
| ensureFieldNames(c.m.vc, "value", res.objectType.ValueFields) | ||
|
|
||
| return res, nil | ||
| } | ||
|
|
||
| // ensureFieldNames makes sure that all fields have valid names - either the | ||
| // names were specified by user or they get filled | ||
| func ensureFieldNames(x any, defaultName string, cols []schema.Field) { | ||
| var names []string = nil | ||
| if hasName, ok := x.(interface{ Name() string }); ok { | ||
| name := hasName.Name() | ||
| if name != "" { | ||
| names = strings.Split(hasName.Name(), ",") | ||
| } | ||
| } | ||
| for i, col := range cols { | ||
| if names != nil && i < len(names) { | ||
| col.Name = names[i] | ||
| } else { | ||
| if col.Name == "" { | ||
| if i == 0 && len(cols) == 1 { | ||
| col.Name = defaultName | ||
| } else { | ||
| col.Name = fmt.Sprintf("%s%d", defaultName, i+1) | ||
| } | ||
| } | ||
| } | ||
| cols[i] = col | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Address undefined types and methods.
The function provides a comprehensive fallback mechanism. However, the following issues need to be addressed:
schema.Typeis undefined.schema.NewModuleSchemais undefined.update.Removeis undefined inschema.KVPairUpdate.Ensure these types and methods are correctly defined or imported.