Events parsing v2 milestone 2 (#338)

* milestone: Add type registry and test

* milestone1: Add metadatas from multiple chains and docker file for tests

* events: Add decoders for each event field type

* registry: Add decoders

* registry: Add retryable executor and use it in parser

* registry: Add mocks and more tests

* registry: Use BitVec when parsing bit sequences

* test: Add more mocks and tests

* registry: Update field name retrieval

* registry-test: Get headers instead of blocks

* Add generic chain RPC, more tests, and Dockerfiles for the 2nd milestone

* make: Add container name

* chain: Add constructor for default chain

* registry: Change field separator to

* parser: Add DefaultExtrinsicParser and DefaultExtrinsic

* retriever: Add DefaultExtrinsicRetriever and adjust tests

* rpc: Add more comments to default entities

* retriever: Enable all live tests

Dockerfile_milestone2

@@ -0,0 +1,7 @@
+FROM golang:1.18
+
+COPY . /go-substrate-rpc-client/events-parsing-v2/milestone-2
+
+WORKDIR /go-substrate-rpc-client/events-parsing-v2/milestone-2
+
+CMD go test -v ./registry/... --cover

Dockerfile_milestone2_live

@@ -0,0 +1,7 @@
+FROM golang:1.18
+
+COPY . /go-substrate-rpc-client/events-parsing-v2/milestone-2
+
+WORKDIR /go-substrate-rpc-client/events-parsing-v2/milestone-2
+
+CMD go test -v -tags=live ./registry/retriever/...

Makefile

@@ -46,9 +46,13 @@ test-types-decode:      ## run tests for types decode
 generate-mocks:      ## generate mocks
 	@docker run -v `pwd`:/app -w /app --entrypoint /bin/sh vektra/mockery:v2.13.0-beta.1 -c 'go generate ./...'
 
-test-milestone1:
-	@docker build -t gsrpc-m1 -f Dockerfile_milestone1 .
-	@docker run --rm gsrpc-m1
+test-milestone2:
+	@docker build -t gsrpc-m2 -f Dockerfile_milestone2 .
+	@docker run --rm gsrpc-m2 --name gsrpc-m2
+
+test-milestone2-live:
+	@docker build -t gsrpc-m2-live -f Dockerfile_milestone2_live .
+	@docker run --rm gsrpc-m2-live --name gsrpc-m2-live
 
 help: 				## shows this help
 	@sed -ne '/@sed/!s/## //p' $(MAKEFILE_LIST)

error/error.go

@@ -0,0 +1,26 @@
+package error
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Error string
+
+func (e Error) Error() string {
+	return string(e)
+}
+
+func (e Error) Is(err error) bool {
+	return strings.Contains(string(e), err.Error())
+}
+
+func (e Error) Wrap(err error) Error {
+	return Error(fmt.Errorf("%s: %w", e, err).Error())
+}
+
+func (e Error) WithMsg(msgFormat string, formatArgs ...any) Error {
+	msg := fmt.Sprintf(msgFormat, formatArgs...)
+
+	return Error(fmt.Sprintf("%s: %s", e, msg))
+}

error/error_test.go

@@ -0,0 +1,36 @@
+package error
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testErr = Error("test error")
+)
+
+func TestError(t *testing.T) {
+	newStdErr := errors.New("new std error")
+	wrappedErr := testErr.Wrap(newStdErr)
+
+	assert.True(t, errors.Is(wrappedErr, testErr))
+	assert.True(t, errors.Is(wrappedErr, newStdErr))
+	assert.Equal(t, fmt.Sprintf("%s: %s", testErr.Error(), newStdErr.Error()), wrappedErr.Error())
+
+	newErr := Error("new error")
+	newWrappedErr := newErr.Wrap(wrappedErr)
+
+	assert.True(t, errors.Is(newWrappedErr, newErr))
+	assert.True(t, errors.Is(newWrappedErr, testErr))
+	assert.True(t, errors.Is(newWrappedErr, newStdErr))
+	assert.Equal(t, fmt.Sprintf("%s: %s", newErr.Error(), wrappedErr.Error()), newWrappedErr.Error())
+
+	err := testErr.WithMsg("%d", 1)
+	assert.Equal(t, fmt.Sprintf("%s: 1", testErr), err.Error())
+
+	err = testErr.WithMsg("test msg")
+	assert.Equal(t, fmt.Sprintf("%s: test msg", testErr), err.Error())
+}

registry/error.go

@@ -0,0 +1,51 @@
+package registry
+
+import libErr "github.com/centrifuge/go-substrate-rpc-client/v4/error"
+
+const (
+	ErrRecursiveDecodersResolving            = libErr.Error("recursive decoders resolving")
+	ErrErrorsTypeNotFound                    = libErr.Error("errors type not found")
+	ErrErrorsTypeNotVariant                  = libErr.Error("errors type not a variant")
+	ErrErrorFieldsRetrieval                  = libErr.Error("error fields retrieval")
+	ErrCallsTypeNotFound                     = libErr.Error("calls type not found")
+	ErrCallsTypeNotVariant                   = libErr.Error("calls type not a variant")
+	ErrCallFieldsRetrieval                   = libErr.Error("call fields retrieval")
+	ErrEventsTypeNotFound                    = libErr.Error("events type not found")
+	ErrEventsTypeNotVariant                  = libErr.Error("events type not a variant")
+	ErrEventFieldsRetrieval                  = libErr.Error("event fields retrieval")
+	ErrFieldDecoderForRecursiveFieldNotFound = libErr.Error("field decoder for recursive field not found")
+	ErrRecursiveFieldResolving               = libErr.Error("recursive field resolving")
+	ErrFieldTypeNotFound                     = libErr.Error("field type not found")
+	ErrFieldDecoderRetrieval                 = libErr.Error("field decoder retrieval")
+	ErrCompactFieldTypeNotFound              = libErr.Error("compact field type not found")
+	ErrCompositeTypeFieldsRetrieval          = libErr.Error("composite type fields retrieval")
+	ErrArrayFieldTypeNotFound                = libErr.Error("array field type not found")
+	ErrVectorFieldTypeNotFound               = libErr.Error("vector field type not found")
+	ErrFieldTypeDefinitionNotSupported       = libErr.Error("field type definition not supported")
+	ErrVariantTypeFieldsRetrieval            = libErr.Error("variant type fields decoding")
+	ErrCompactTupleItemTypeNotFound          = libErr.Error("compact tuple item type not found")
+	ErrCompactTupleItemFieldDecoderRetrieval = libErr.Error("compact tuple item field decoder retrieval")
+	ErrCompactCompositeFieldTypeNotFound     = libErr.Error("compact composite field type not found")
+	ErrCompactCompositeFieldDecoderRetrieval = libErr.Error("compact composite field decoder retrieval")
+	ErrArrayItemFieldDecoderRetrieval        = libErr.Error("array item field decoder retrieval")
+	ErrSliceItemFieldDecoderRetrieval        = libErr.Error("slice item field decoder retrieval")
+	ErrTupleItemTypeNotFound                 = libErr.Error("tuple item type not found")
+	ErrTupleItemFieldDecoderRetrieval        = libErr.Error("tuple item field decoder retrieval")
+	ErrBitStoreTypeNotFound                  = libErr.Error("bit store type not found")
+	ErrBitStoreTypeNotSupported              = libErr.Error("bit store type not supported")
+	ErrBitOrderTypeNotFound                  = libErr.Error("bit order type not found")
+	ErrBitOrderCreation                      = libErr.Error("bit order creation")
+	ErrPrimitiveTypeNotSupported             = libErr.Error("primitive type not supported")
+	ErrTypeFieldDecoding                     = libErr.Error("type field decoding")
+	ErrVariantByteDecoding                   = libErr.Error("variant byte decoding")
+	ErrVariantFieldDecoderNotFound           = libErr.Error("variant field decoder not found")
+	ErrArrayItemDecoderNotFound              = libErr.Error("array item decoder not found")
+	ErrArrayItemDecoding                     = libErr.Error("array item decoding")
+	ErrSliceItemDecoderNotFound              = libErr.Error("slice item decoder not found")
+	ErrSliceLengthDecoding                   = libErr.Error("slice length decoding")
+	ErrSliceItemDecoding                     = libErr.Error("slice item decoding")
+	ErrCompositeFieldDecoding                = libErr.Error("composite field decoding")
+	ErrValueDecoding                         = libErr.Error("value decoding")
+	ErrRecursiveFieldDecoderNotFound         = libErr.Error("recursive field decoder not found")
+	ErrBitVecDecoding                        = libErr.Error("bit vec decoding")
+)

registry/exec/exec.go

@@ -0,0 +1,161 @@
+package exec
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+)
+
+//go:generate mockery --name RetryableExecutor --structname RetryableExecutorMock --filename exec_mock.go --inpackage
+
+// RetryableExecutor is the interface used for executing a closure and its fallback if the initial execution fails.
+//
+// The interface is generic over type T which represents the return value of the closure.
+type RetryableExecutor[T any] interface {
+	ExecWithFallback(execFn func() (T, error), fallbackFn func() error) (T, error)
+}
+
+// retryableExecutor implements RetryableExecutor.
+//
+// It can be configured via the provided OptsFn(s).
+type retryableExecutor[T any] struct {
+	opts *Opts
+}
+
+// NewRetryableExecutor creates a new RetryableExecutor.
+func NewRetryableExecutor[T any](opts ...OptsFn) RetryableExecutor[T] {
+	execOpts := NewDefaultExecOpts()
+
+	for _, opt := range opts {
+		opt(execOpts)
+	}
+
+	return &retryableExecutor[T]{
+		execOpts,
+	}
+}
+
+// ExecWithFallback will attempt to execute the provided execFn and, in the case of failure, it will execute
+// the fallbackFn and retry execution of execFn.
+func (r *retryableExecutor[T]) ExecWithFallback(execFn func() (T, error), fallbackFn func() error) (res T, err error) {
+	if execFn == nil {
+		return res, ErrMissingExecFn
+	}
+
+	if fallbackFn == nil {
+		return res, ErrMissingFallbackFn
+	}
+
+	execErr := &Error{}
+
+	retryCount := uint(0)
+
+	for {
+		res, err = execFn()
+
+		if err == nil {
+			return res, nil
+		}
+
+		execErr.AddErr(fmt.Errorf("exec function error: %w", err))
+
+		if retryCount == r.opts.maxRetryCount {
+			return res, execErr
+		}
+
+		if err = fallbackFn(); err != nil && !r.opts.retryOnFallbackError {
+			execErr.AddErr(fmt.Errorf("fallback function error: %w", err))
+
+			return res, execErr
+		}
+
+		retryCount++
+
+		time.Sleep(r.opts.retryTimeout)
+	}
+}
+
+var (
+	ErrMissingExecFn     = errors.New("no exec function provided")
+	ErrMissingFallbackFn = errors.New("no fallback function provided")
+)
+
+const (
+	defaultMaxRetryCount        = 3
+	defaultErrTimeout           = 0 * time.Second
+	defaultRetryOnFallbackError = true
+)
+
+// Opts holds the configurable options for a RetryableExecutor.
+type Opts struct {
+	// maxRetryCount holds maximum number of retries in the case of failure.
+	maxRetryCount uint
+
+	// retryTimeout holds the timeout between retries.
+	retryTimeout time.Duration
+
+	// retryOnFallbackError specifies whether a retry will be done in the case of
+	// failure of the fallback function.
+	retryOnFallbackError bool
+}
+
+// NewDefaultExecOpts creates the default Opts.
+func NewDefaultExecOpts() *Opts {
+	return &Opts{
+		maxRetryCount:        defaultMaxRetryCount,
+		retryTimeout:         defaultErrTimeout,
+		retryOnFallbackError: defaultRetryOnFallbackError,
+	}
+}
+
+// OptsFn is function that operate on Opts.
+type OptsFn func(opts *Opts)
+
+// WithMaxRetryCount sets the max retry count.
+//
+// Note that a default value is provided if the provided count is 0.
+func WithMaxRetryCount(maxRetryCount uint) OptsFn {
+	return func(opts *Opts) {
+		if maxRetryCount == 0 {
+			maxRetryCount = defaultMaxRetryCount
+		}
+
+		opts.maxRetryCount = maxRetryCount
+	}
+}
+
+// WithRetryTimeout sets the retry timeout.
+func WithRetryTimeout(retryTimeout time.Duration) OptsFn {
+	return func(opts *Opts) {
+		opts.retryTimeout = retryTimeout
+	}
+}
+
+// WithRetryOnFallBackError sets the retryOnFallbackError flag.
+func WithRetryOnFallBackError(retryOnFallbackError bool) OptsFn {
+	return func(opts *Opts) {
+		opts.retryOnFallbackError = retryOnFallbackError
+	}
+}
+
+// Error holds none or multiple errors that can happen during execution.
+type Error struct {
+	errs []error
+}
+
+// AddErr appends an error to the error slice of Error.
+func (e *Error) AddErr(err error) {
+	e.errs = append(e.errs, err)
+}
+
+// Error implements the standard error interface.
+func (e *Error) Error() string {
+	sb := strings.Builder{}
+
+	for i, err := range e.errs {
+		sb.WriteString(fmt.Sprintf("error %d: %s\n", i, err))
+	}
+
+	return sb.String()
+}