EIP-4844

.github/workflows/test-spec.yml

@@ -59,5 +59,5 @@ jobs:
         run: yarn test:spec-minimal
         working-directory: packages/beacon-node
       - name: Spec tests mainnet
-        run: yarn test:spec-mainnet
+        run: yarn test:spec-main
         working-directory: packages/beacon-node

CONTRIBUTING.md

@@ -16,10 +16,13 @@ Thanks for your contribution to Lodestar. It's people like you that push the Eth
 
 ## Tests
 
+- :computer: Run `lerna run download-spec-tests` to download the files necessary to run spec tests.
 - :test_tube: Run `lerna run test:spec-min` for minimal spec tests.
 - :test_tube: Run `lerna run test:spec-main` for mainnet spec tests.
 - :test_tube: Run `lerna run test:unit` for unit tests.
 - :test_tube: Run `lerna run test:e2e` for end-to-end tests.
+- :test_tube: Run `yarn check-types` to check TypeScript types.
+- :test_tube: Run `yarn lint` to run the linter (ESLint).
 - :test_tube: Run `lerna run test` to run all tests.
 
 ### Debugging Spec Tests
@@ -101,7 +104,7 @@ For example:
 
 > Add Edit on Github button to all the pages
 
-**Pull Request Etiquette** 
+**Pull Request Etiquette**
 
 - Pull requests should remain as drafts when they are not ready for review by maintainers. Open pull requests signal to the maintainers that it's ready for review.
 - If your pull request is no longer applicable or validated to fix an issue, close your pull request.

package.json

@@ -40,7 +40,7 @@
     "@types/chai": "^4.3.3",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.0",
-    "@types/node": "^18.7.23",
+    "@types/node": "^18.11.9",
     "@types/sinon": "^10.0.13",
     "@types/sinon-chai": "^3.2.8",
     "@typescript-eslint/eslint-plugin": "5.26.0",
@@ -51,10 +51,10 @@
     "crypto-browserify": "^3.12.0",
     "electron": "^21.0.1",
     "eslint": "^8.16.0",
-    "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-prettier": "^4.0.0",
     "eslint-plugin-chai-expect": "^3.0.0",
+    "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-mocha": "^10.1.0",
+    "eslint-plugin-prettier": "^4.0.0",
     "https-browserify": "^1.0.0",
     "karma": "^6.4.1",
     "karma-chai": "^0.1.0",

packages/api/src/beacon/routes/beacon/block.ts

@@ -2,6 +2,7 @@ import {ContainerType} from "@chainsafe/ssz";
 import {ForkName} from "@lodestar/params";
 import {IChainForkConfig} from "@lodestar/config";
 import {phase0, allForks, Slot, Root, ssz, RootHex} from "@lodestar/types";
+import {SignedBeaconBlockAndBlobsSidecar} from "@lodestar/types/eip4844";
 
 import {
   RoutesData,
@@ -104,11 +105,14 @@ export type Api = {
    * @returns any The block was validated successfully and has been broadcast. It has also been integrated into the beacon node's database.
    */
   publishBlock(block: allForks.SignedBeaconBlock): Promise<void>;
+
   /**
    * Publish a signed blinded block by submitting it to the mev relay and patching in the block
    * transactions beacon node gets in response.
    */
   publishBlindedBlock(block: allForks.SignedBlindedBeaconBlock): Promise<void>;
+
+  publishBlockWithBlobs(blockWithBlobs: SignedBeaconBlockAndBlobsSidecar): Promise<void>;
 };
 
 /**
@@ -122,6 +126,7 @@ export const routesData: RoutesData<Api> = {
   getBlockHeaders: {url: "/eth/v1/beacon/headers", method: "GET"},
   getBlockRoot: {url: "/eth/v1/beacon/blocks/{block_id}/root", method: "GET"},
   publishBlock: {url: "/eth/v1/beacon/blocks", method: "POST"},
+  publishBlockWithBlobs: {url: "/eth/v1/beacon/blocksWithBlobs", method: "POST"},
   publishBlindedBlock: {url: "/eth/v1/beacon/blinded_blocks", method: "POST"},
 };
 
@@ -137,6 +142,7 @@ export type ReqTypes = {
   getBlockHeaders: {query: {slot?: number; parent_root?: string}};
   getBlockRoot: BlockIdOnlyReq;
   publishBlock: {body: unknown};
+  publishBlockWithBlobs: {body: unknown};
   publishBlindedBlock: {body: unknown};
 };
 
@@ -156,6 +162,12 @@ export function getReqSerializers(config: IChainForkConfig): ReqSerializers<Api,
     fromJson: (data) => getSignedBeaconBlockType((data as unknown) as allForks.SignedBeaconBlock).fromJson(data),
   };
 
+  const BlockWithBlobsSsz = ssz.eip4844.SignedBeaconBlockAndBlobsSidecar;
+  const SignedBeaconBlockWithBlobs: TypeJson<SignedBeaconBlockAndBlobsSidecar> = {
+    toJson: (data) => BlockWithBlobsSsz.toJson(data),
+    fromJson: (data) => BlockWithBlobsSsz.fromJson(data),
+  };
+
   const getSignedBlindedBeaconBlockType = (
     data: allForks.SignedBlindedBeaconBlock
   ): allForks.AllForksBlindedSSZTypes["SignedBeaconBlock"] =>
@@ -179,6 +191,7 @@ export function getReqSerializers(config: IChainForkConfig): ReqSerializers<Api,
     },
     getBlockRoot: blockIdOnlyReq,
     publishBlock: reqOnlyBody(AllForksSignedBeaconBlock, Schema.Object),
+    publishBlockWithBlobs: reqOnlyBody(SignedBeaconBlockWithBlobs, Schema.Object),
     publishBlindedBlock: reqOnlyBody(AllForksSignedBlindedBeaconBlock, Schema.Object),
   };
 }

packages/api/src/beacon/routes/validator.ts

@@ -9,6 +9,7 @@ import {
   Epoch,
   phase0,
   bellatrix,
+  eip4844,
   Root,
   Slot,
   ssz,
@@ -23,6 +24,7 @@ import {
   ArrayOf,
   ContainerData,
   Schema,
+  WithBlobs,
   WithVersion,
   reqOnlyBody,
   ReqSerializers,
@@ -165,6 +167,12 @@ export type Api = {
     graffiti: string
   ): Promise<{data: allForks.BeaconBlock; version: ForkName}>;
 
+  produceBlockWithBlobs(
+    slot: Slot,
+    randaoReveal: BLSSignature,
+    graffiti: string
+  ): Promise<{data: allForks.BeaconBlock; version: ForkName; blobs: eip4844.Blobs}>;
+
   produceBlindedBlock(
     slot: Slot,
     randaoReveal: BLSSignature,
@@ -245,6 +253,7 @@ export const routesData: RoutesData<Api> = {
   getSyncCommitteeDuties: {url: "/eth/v1/validator/duties/sync/{epoch}", method: "POST"},
   produceBlock: {url: "/eth/v1/validator/blocks/{slot}", method: "GET"},
   produceBlockV2: {url: "/eth/v2/validator/blocks/{slot}", method: "GET"},
+  produceBlockWithBlobs: {url: "/eth/v3/validator/blocks/{slot}", method: "GET"},
   produceBlindedBlock: {url: "/eth/v1/validator/blinded_blocks/{slot}", method: "GET"},
   produceAttestationData: {url: "/eth/v1/validator/attestation_data", method: "GET"},
   produceSyncCommitteeContribution: {url: "/eth/v1/validator/sync_committee_contribution", method: "GET"},
@@ -265,6 +274,7 @@ export type ReqTypes = {
   getSyncCommitteeDuties: {params: {epoch: Epoch}; body: U64Str[]};
   produceBlock: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}};
   produceBlockV2: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}};
+  produceBlockWithBlobs: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}};
   produceBlindedBlock: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}};
   produceAttestationData: {query: {slot: number; committee_index: number}};
   produceSyncCommitteeContribution: {query: {slot: number; subcommittee_index: number; beacon_block_root: string}};
@@ -340,6 +350,7 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
 
     produceBlock: produceBlock,
     produceBlockV2: produceBlock,
+    produceBlockWithBlobs: produceBlock,
     produceBlindedBlock: produceBlock,
 
     produceAttestationData: {
@@ -443,6 +454,7 @@ export function getReturnTypes(): ReturnTypes<Api> {
     getSyncCommitteeDuties: ContainerDataExecutionOptimistic(ArrayOf(SyncDuty)),
     produceBlock: ContainerData(ssz.phase0.BeaconBlock),
     produceBlockV2: WithVersion((fork: ForkName) => ssz[fork].BeaconBlock),
+    produceBlockWithBlobs: WithBlobs(WithVersion((fork: ForkName) => ssz[fork].BeaconBlock)),
     produceBlindedBlock: WithVersion((fork: ForkName) => {
       if (fork === ForkName.phase0 || fork === ForkName.altair) {
         throw Error(`No BlindedBlock for fork ${fork} previous to bellatrix`);

packages/api/src/utils/types.ts

@@ -2,6 +2,8 @@ import {isBasicType, ListBasicType, Type, isCompositeType, ListCompositeType, Ar
 import {ForkName} from "@lodestar/params";
 import {IChainForkConfig} from "@lodestar/config";
 import {objectToExpectedCase} from "@lodestar/utils";
+import {ssz} from "@lodestar/types";
+import {Blobs} from "@lodestar/types/eip4844";
 import {Schema, SchemaDefinition} from "./schema.js";
 
 // See /packages/api/src/routes/index.ts for reasoning
@@ -195,6 +197,19 @@ export function WithExecutionOptimistic<T extends {data: unknown}>(
   };
 }
 
+export function WithBlobs<T extends {data: unknown}>(type: TypeJson<T>): TypeJson<T & {blobs: Blobs}> {
+  return {
+    toJson: ({blobs, ...data}) => ({
+      ...(type.toJson((data as unknown) as T) as Record<string, unknown>),
+      blobs: ssz.eip4844.Blobs.toJson(blobs),
+    }),
+    fromJson: ({blobs, ...data}: T & {blobs: Blobs}) => ({
+      ...type.fromJson(data),
+      blobs: ssz.eip4844.Blobs.fromJson(blobs),
+    }),
+  };
+}
+
 type JsonCase = "snake" | "constant" | "camel" | "param" | "header" | "pascal" | "dot" | "notransform";
 
 /** Helper to only translate casing */

packages/beacon-node/package.json

@@ -130,6 +130,7 @@
     "@multiformats/multiaddr": "^11.0.0",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",
+    "c-kzg": "^1.0.4",
     "cross-fetch": "^3.1.4",
     "datastore-core": "^8.0.1",
     "datastore-level": "^9.0.1",
@@ -158,6 +159,7 @@
     "@types/eventsource": "^1.1.5",
     "@types/it-all": "^1.0.0",
     "@types/leveldown": "^4.0.2",
+    "@types/node": "^18.11.9",
     "@types/prometheus-gc-stats": "^0.6.1",
     "@types/supertest": "^2.0.12",
     "@types/tmp": "^0.2.0",

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -23,6 +23,16 @@ export function getBeaconBlockApi({
   network,
   db,
 }: Pick<ApiModules, "chain" | "config" | "metrics" | "network" | "db">): routes.beacon.block.Api {
+  const waitForSlot = async (slot: number): Promise<void> => {
+    // Simple implementation of a pending block queue. Keeping the block here recycles the API logic, and keeps the
+    // REST request promise without any extra infrastructure.
+    const msToBlockSlot = computeTimeAtSlot(config, slot, chain.genesisTime) * 1000 - Date.now();
+    if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) {
+      // If block is a bit early, hold it in a promise. Equivalent to a pending queue.
+      await sleep(msToBlockSlot);
+    }
+  };
+
   return {
     async getBlockHeaders(filters) {
       // TODO - SLOW CODE: This code seems like it could be improved
@@ -173,14 +183,7 @@ export function getBeaconBlockApi({
 
     async publishBlock(signedBlock) {
       const seenTimestampSec = Date.now() / 1000;
-
-      // Simple implementation of a pending block queue. Keeping the block here recycles the API logic, and keeps the
-      // REST request promise without any extra infrastructure.
-      const msToBlockSlot = computeTimeAtSlot(config, signedBlock.message.slot, chain.genesisTime) * 1000 - Date.now();
-      if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) {
-        // If block is a bit early, hold it in a promise. Equivalent to a pending queue.
-        await sleep(msToBlockSlot);
-      }
+      await waitForSlot(signedBlock.message.slot);
 
       // TODO: Validate block
 
@@ -199,5 +202,29 @@ export function getBeaconBlockApi({
         }),
       ]);
     },
+
+    async publishBlockWithBlobs(signedBeaconBlockAndBlobsSidecar) {
+      const {beaconBlock, blobsSidecar} = signedBeaconBlockAndBlobsSidecar;
+      const {message} = beaconBlock;
+
+      const seenTimestampSec = Date.now() / 1000;
+      await waitForSlot(message.slot);
+
+      metrics?.registerBeaconBlock(OpSource.api, seenTimestampSec, message);
+
+      await db.blobsSidecar.add(blobsSidecar);
+
+      await Promise.all([
+        network.gossip.publishSignedBeaconBlockAndBlobsSidecar(signedBeaconBlockAndBlobsSidecar),
+        // TODO EIP-4844 processBlock for signedBeaconBlockAndBlobsSidecar
+        // We need to save the blob?
+        chain.processBlock(beaconBlock).catch((e) => {
+          if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) {
+            network.events.emit(NetworkEvent.unknownBlockParent, beaconBlock, network.peerId.toString());
+          }
+          throw e;
+        }),
+      ]);
+    },
   };
 }

packages/beacon-node/src/api/impl/beacon/state/utils.ts

@@ -221,10 +221,11 @@ async function getFinalizedState(
 
   // process blocks up to the requested slot
   for await (const block of db.blockArchive.valuesStream({gt: state.slot, lte: slot})) {
-    state = stateTransition(state, block, {
+    state = stateTransition(state, block, undefined, {
       verifyStateRoot: false,
       verifyProposer: false,
       verifySignatures: false,
+      verifyBlobs: false,
     });
     // yield to the event loop
     await sleep(0);

packages/beacon-node/src/api/impl/config/constants.ts

@@ -19,6 +19,7 @@ import {
   DOMAIN_CONTRIBUTION_AND_PROOF,
   DOMAIN_BLS_TO_EXECUTION_CHANGE,
   DOMAIN_APPLICATION_BUILDER,
+  DOMAIN_BLOBS_SIDECAR,
   TIMELY_SOURCE_FLAG_INDEX,
   TIMELY_TARGET_FLAG_INDEX,
   TIMELY_HEAD_FLAG_INDEX,
@@ -94,4 +95,7 @@ export const specConstants = {
 
   // ## Capella domain types
   DOMAIN_BLS_TO_EXECUTION_CHANGE,
+
+  // ## EIP-4844 domain types
+  DOMAIN_BLOBS_SIDECAR,
 };

packages/beacon-node/src/api/impl/validator/index.ts

@@ -238,9 +238,40 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}:
     }
   };
 
+  const produceBlockWithBlobs: routes.validator.Api["produceBlockWithBlobs"] = async function produceBlock(
+    slot,
+    randaoReveal,
+    graffiti
+  ) {
+    let timer;
+    metrics?.blockProductionRequests.inc();
+    try {
+      notWhileSyncing();
+      await waitForSlot(slot); // Must never request for a future slot > currentSlot
+
+      // Process the queued attestations in the forkchoice for correct head estimation
+      // forkChoice.updateTime() might have already been called by the onSlot clock
+      // handler, in which case this should just return.
+      chain.forkChoice.updateTime(slot);
+      chain.recomputeForkChoiceHead();
+
+      timer = metrics?.blockProductionTime.startTimer();
+      const {block, blobs} = await chain.produceBlockWithBlobs({
+        slot,
+        randaoReveal,
+        graffiti: toGraffitiBuffer(graffiti || ""),
+      });
+      metrics?.blockProductionSuccess.inc();
+      return {data: block, version: config.getForkName(block.slot), blobs};
+    } finally {
+      if (timer) timer();
+    }
+  };
+
   return {
     produceBlock: produceBlock,
     produceBlockV2: produceBlock,
+    produceBlockWithBlobs,
     produceBlindedBlock,
 
     async produceAttestationData(committeeIndex, slot) {

packages/beacon-node/src/chain/blocks/index.ts

@@ -13,7 +13,7 @@ import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {verifyBlocksSanityChecks} from "./verifyBlocksSanityChecks.js";
 export {ImportBlockOpts} from "./types.js";
 
-const QUEUE_MAX_LENGHT = 256;
+const QUEUE_MAX_LENGTH = 256;
 
 /**
  * BlockProcessor processes block jobs in a queued fashion, one after the other.
@@ -26,7 +26,7 @@ export class BlockProcessor {
       (job, importOpts) => {
         return processBlocks.call(chain, job, {...opts, ...importOpts});
       },
-      {maxLength: QUEUE_MAX_LENGHT, signal},
+      {maxLength: QUEUE_MAX_LENGTH, signal},
       metrics?.blockProcessorQueue ?? undefined
     );
   }

packages/beacon-node/src/chain/blocks/verifyBlock.ts

@@ -67,7 +67,15 @@ export async function verifyBlocksInEpoch(
     const [{postStates, proposerBalanceDeltas}, , segmentExecStatus] = await Promise.all([
       // Run state transition only
       // TODO: Ensure it yields to allow flushing to workers and engine API
-      verifyBlocksStateTransitionOnly(preState0, blocks, this.metrics, abortController.signal, opts),
+      verifyBlocksStateTransitionOnly(
+        preState0,
+        blocks,
+        this.metrics,
+        this.config,
+        this.db,
+        abortController.signal,
+        opts
+      ),
 
       // All signatures at once
       verifyBlocksSignatures(this.bls, preState0, blocks, opts),