Merge branch 'main' into persisted-peerstore

.github/workflows/ci_release.yml

@@ -24,15 +24,15 @@ on:
 jobs:
   # Dockerfile Linting
   hadolint:
-    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.1.1 # yamllint disable-line rule:line-length
+    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.0 # yamllint disable-line rule:line-length
     with:
       dockerfile: Dockerfile
 
   yamllint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: celestiaorg/.github/.github/actions/yamllint@v0.1.1
+      - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.0
 
   markdown-lint:
     name: Markdown Lint
@@ -58,7 +58,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Version Release
-        uses: celestiaorg/.github/.github/actions/version-release@v0.1.1
+        uses: celestiaorg/.github/.github/actions/version-release@v0.2.0
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           version-bump: ${{inputs.version}}

.github/workflows/docker-build-publish.yml

@@ -17,6 +17,6 @@ jobs:
     permissions:
       contents: write
       packages: write
-    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.1.1 # yamllint disable-line rule:line-length
+    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.0 # yamllint disable-line rule:line-length
     with:
       dockerfile: Dockerfile

.github/workflows/go-ci.yml

@@ -23,7 +23,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.4.0
+        uses: golangci/golangci-lint-action@v3.6.0
         with:
           version: v1.52.2
 
@@ -59,7 +59,7 @@ jobs:
         run: make test-unit
 
       - name: upload coverage
-        uses: codecov/codecov-action@v3.1.3
+        uses: codecov/codecov-action@v3.1.4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.txt

.github/workflows/labels.yml

@@ -12,7 +12,7 @@ jobs:
   label:
     runs-on: ubuntu-latest
     steps:
-      - uses: mheap/github-action-required-labels@v4
+      - uses: mheap/github-action-required-labels@v5
         with:
           mode: minimum
           count: 1

Dockerfile

@@ -15,7 +15,7 @@ COPY . .
 
 RUN make build && make cel-key
 
-FROM docker.io/alpine:3.17.3
+FROM docker.io/alpine:3.18.0
 
 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000
 ARG UID=10001

Makefile

@@ -152,7 +152,7 @@ pb-gen:
 ## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api
 openrpc-gen:
 	@echo "--> Generating OpenRPC spec"
-	@go run ./cmd/docgen fraud header state share das p2p node
+	@go run ./cmd/docgen fraud header state share das p2p node blob
 .PHONY: openrpc-gen
 
 ## lint-imports: Lint only Go imports.

api/docgen/examples.go

@@ -18,8 +18,10 @@ import (
 	"golang.org/x/text/language"
 
 	"github.com/celestiaorg/go-fraud"
+	"github.com/celestiaorg/nmt"
 	"github.com/celestiaorg/rsmt2d"
 
+	"github.com/celestiaorg/celestia-node/blob"
 	"github.com/celestiaorg/celestia-node/das"
 	"github.com/celestiaorg/celestia-node/header"
 	"github.com/celestiaorg/celestia-node/nodebuilder/node"
@@ -81,6 +83,8 @@ func init() {
 	}
 	addToExampleValues(valAddr)
 
+	addToExampleValues(state.Address{Address: addr})
+
 	var txResponse *state.TxResponse
 	err = json.Unmarshal([]byte(exampleTxResponse), &txResponse)
 	if err != nil {
@@ -128,6 +132,22 @@ func init() {
 		Addrs: []multiaddr.Multiaddr{ma},
 	}
 	addToExampleValues(addrInfo)
+
+	namespace, err := share.NewNamespaceV0([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10})
+	if err != nil {
+		panic(err)
+	}
+	addToExampleValues(namespace)
+
+	generatedBlob, err := blob.NewBlob(0, namespace, []byte("This is an example of some blob data"))
+	if err != nil {
+		panic(err)
+	}
+	addToExampleValues(generatedBlob)
+
+	proof := nmt.NewInclusionProof(0, 4, [][]byte{[]byte("test")}, true)
+	blobProof := &blob.Proof{&proof}
+	addToExampleValues(blobProof)
 }
 
 func addToExampleValues(v interface{}) {

api/gateway/availability.go

@@ -2,7 +2,6 @@ package gateway
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"strconv"
 
@@ -28,20 +27,6 @@ func (h *Handler) handleHeightAvailabilityRequest(w http.ResponseWriter, r *http
 		return
 	}
 
-	//TODO: change this to NetworkHead once the adjacency in the store is fixed.
-	head, err := h.header.LocalHead(r.Context())
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err)
-		return
-	}
-	if headHeight := int(head.Height()); headHeight < height {
-		err = fmt.Errorf(
-			"current head local chain head: %d is lower than requested height: %d"+
-				" give header sync some time and retry later", headHeight, height)
-		writeError(w, http.StatusServiceUnavailable, heightAvailabilityEndpoint, err)
-		return
-	}
-
 	header, err := h.header.GetByHeight(r.Context(), uint64(height))
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err)

api/gateway/das.go

@@ -10,6 +10,7 @@ const (
 )
 
 func (h *Handler) handleDASStateRequest(w http.ResponseWriter, r *http.Request) {
+	logDeprecation(dasStateEndpoint, "das.SamplingStats")
 	stats, err := h.das.SamplingStats(r.Context())
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, dasStateEndpoint, err)

api/gateway/endpoints.go

@@ -5,21 +5,32 @@ import (
 	"net/http"
 )
 
-func (h *Handler) RegisterEndpoints(rpc *Server) {
+func (h *Handler) RegisterEndpoints(rpc *Server, deprecatedEndpointsEnabled bool) {
+	if deprecatedEndpointsEnabled {
+		log.Warn("Deprecated endpoints will be removed from the gateway in the next release. Use the RPC instead.")
+		// state endpoints
+		rpc.RegisterHandlerFunc(balanceEndpoint, h.handleBalanceRequest, http.MethodGet)
+		rpc.RegisterHandlerFunc(submitPFBEndpoint, h.handleSubmitPFB, http.MethodPost)
+
+		// staking queries
+		rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryDelegationEndpoint, addrKey), h.handleQueryDelegation,
+			http.MethodGet)
+		rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryUnbondingEndpoint, addrKey), h.handleQueryUnbonding,
+			http.MethodGet)
+		rpc.RegisterHandlerFunc(queryRedelegationsEndpoint, h.handleQueryRedelegations,
+			http.MethodPost)
+
+		// DASer endpoints
+		// only register if DASer service is available
+		if h.das != nil {
+			rpc.RegisterHandlerFunc(dasStateEndpoint, h.handleDASStateRequest, http.MethodGet)
+		}
+	}
+
 	// state endpoints
-	rpc.RegisterHandlerFunc(balanceEndpoint, h.handleBalanceRequest, http.MethodGet)
 	rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", balanceEndpoint, addrKey), h.handleBalanceRequest,
 		http.MethodGet)
 	rpc.RegisterHandlerFunc(submitTxEndpoint, h.handleSubmitTx, http.MethodPost)
-	rpc.RegisterHandlerFunc(submitPFBEndpoint, h.handleSubmitPFB, http.MethodPost)
-
-	// staking queries
-	rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryDelegationEndpoint, addrKey), h.handleQueryDelegation,
-		http.MethodGet)
-	rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryUnbondingEndpoint, addrKey), h.handleQueryUnbonding,
-		http.MethodGet)
-	rpc.RegisterHandlerFunc(queryRedelegationsEndpoint, h.handleQueryRedelegations,
-		http.MethodPost)
 
 	// share endpoints
 	rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, nIDKey, heightKey),
@@ -39,10 +50,4 @@ func (h *Handler) RegisterEndpoints(rpc *Server) {
 	rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", headerByHeightEndpoint, heightKey), h.handleHeaderRequest,
 		http.MethodGet)
 	rpc.RegisterHandlerFunc(headEndpoint, h.handleHeadRequest, http.MethodGet)
-
-	// DASer endpoints
-	// only register if DASer service is available
-	if h.das != nil {
-		rpc.RegisterHandlerFunc(dasStateEndpoint, h.handleDASStateRequest, http.MethodGet)
-	}
 }

api/gateway/header.go

@@ -2,7 +2,6 @@ package gateway
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"strconv"
 
@@ -70,24 +69,12 @@ func (h *Handler) performGetHeaderRequest(
 		writeError(w, http.StatusBadRequest, endpoint, err)
 		return nil, err
 	}
-	//TODO: change this to NetworkHead once the adjacency in the store is fixed.
-	head, err := h.header.LocalHead(r.Context())
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err)
-		return nil, err
-	}
-	if headHeight := int(head.Height()); headHeight < height {
-		err = fmt.Errorf(
-			"current head local chain head: %d is lower than requested height: %d"+
-				" give header sync some time and retry later", headHeight, height)
-		writeError(w, http.StatusServiceUnavailable, endpoint, err)
-		return nil, err
-	}
-	// perform request
+
 	header, err := h.header.GetByHeight(r.Context(), uint64(height))
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, endpoint, err)
 		return nil, err
 	}
+
 	return header, nil
 }

api/gateway/share.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"strconv"
 
@@ -13,7 +12,6 @@ import (
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/nmt/namespace"
 
-	"github.com/celestiaorg/celestia-node/header"
 	"github.com/celestiaorg/celestia-node/share"
 )
 
@@ -44,14 +42,14 @@ func (h *Handler) handleSharesByNamespaceRequest(w http.ResponseWriter, r *http.
 		writeError(w, http.StatusBadRequest, namespacedSharesEndpoint, err)
 		return
 	}
-	shares, headerHeight, err := h.getShares(r.Context(), height, nID)
+	shares, err := h.getShares(r.Context(), height, nID)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, namespacedSharesEndpoint, err)
 		return
 	}
 	resp, err := json.Marshal(&NamespacedSharesResponse{
 		Shares: shares,
-		Height: uint64(headerHeight),
+		Height: height,
 	})
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, namespacedSharesEndpoint, err)
@@ -69,7 +67,7 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re
 		writeError(w, http.StatusBadRequest, namespacedDataEndpoint, err)
 		return
 	}
-	shares, headerHeight, err := h.getShares(r.Context(), height, nID)
+	shares, err := h.getShares(r.Context(), height, nID)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, namespacedDataEndpoint, err)
 		return
@@ -81,7 +79,7 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re
 	}
 	resp, err := json.Marshal(&NamespacedDataResponse{
 		Data:   data,
-		Height: uint64(headerHeight),
+		Height: height,
 	})
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, namespacedDataEndpoint, err)
@@ -93,33 +91,18 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re
 	}
 }
 
-func (h *Handler) getShares(ctx context.Context, height uint64, nID namespace.ID) ([]share.Share, int64, error) {
-	// get header
-	var (
-		err    error
-		header *header.ExtendedHeader
-	)
-
-	//TODO: change this to NetworkHead once the adjacency in the store is fixed.
-	header, err = h.header.LocalHead(ctx)
+func (h *Handler) getShares(ctx context.Context, height uint64, nID namespace.ID) ([]share.Share, error) {
+	header, err := h.header.GetByHeight(ctx, height)
 	if err != nil {
-		return nil, 0, err
+		return nil, err
 	}
 
-	if height > 0 {
-		if storeHeight := uint64(header.Height()); storeHeight < height {
-			return nil, 0, fmt.Errorf(
-				"current head local chain head: %d is lower than requested height: %d"+
-					" give header sync some time and retry later", storeHeight, height)
-		}
-		header, err = h.header.GetByHeight(ctx, height)
-	}
+	shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, nID)
 	if err != nil {
-		return nil, 0, err
+		return nil, err
 	}
-	// perform request
-	shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, nID)
-	return shares.Flatten(), header.Height(), err
+
+	return shares.Flatten(), nil
 }
 
 func dataFromShares(input []share.Share) (data [][]byte, err error) {

api/gateway/share_test.go

@@ -2,11 +2,8 @@ package gateway
 
 import (
 	_ "embed"
-	"encoding/base64"
-	"encoding/json"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	coretypes "github.com/tendermint/tendermint/types"
 
@@ -48,37 +45,3 @@ func Test_dataFromShares(t *testing.T) {
 
 	require.Equal(t, testData, parsedSSSShares)
 }
-
-// sharesBase64JSON is the base64 encoded share data from Blockspace Race
-// block height 559108 and namespace e8e5f679bf7116cb.
-//
-//go:embed "testdata/sharesBase64.json"
-var sharesBase64JSON string
-
-// Test_dataFromSharesBSR reproduces an error that occurred when parsing shares
-// on Blockspace Race block height 559108 namespace e8e5f679bf7116cb.
-//
-// https://github.com/celestiaorg/celestia-app/issues/1816
-func Test_dataFromSharesBSR(t *testing.T) {
-	t.Skip("skip until sharesBase64JSON is regenerated with v1 compatibility")
-
-	var sharesBase64 []string
-	err := json.Unmarshal([]byte(sharesBase64JSON), &sharesBase64)
-	assert.NoError(t, err)
-	input := decode(sharesBase64)
-
-	_, err = dataFromShares(input)
-	assert.NoError(t, err)
-}
-
-// decode returns the raw shares from base64Encoded.
-func decode(base64Encoded []string) (rawShares [][]byte) {
-	for _, share := range base64Encoded {
-		rawShare, err := base64.StdEncoding.DecodeString(share)
-		if err != nil {
-			panic(err)
-		}
-		rawShares = append(rawShares, rawShare)
-	}
-	return rawShares
-}