Migrate JumpDestCache to CodeCache (#3472)

Migrate the jump dest analysis cache into a more generic code cache that
lives inside the EVM.

This is in preparation for Ethereum Object Formats where a string of
code may be treated differently depending on what EVM version is
executing the code. Newer versions will also have difference analyses to
run that will need different backing data structures.

Signed-off-by: Danno Ferrin <danno.ferrin@gmail.com>

build.gradle

@@ -130,7 +130,7 @@ allprojects {
     java {
       // This path needs to be relative to each project
       target '**/*.java'
-      targetExclude '**/src/reference-test/**', '**/src/main/generated/**', '**/src/test/generated/**'
+      targetExclude '**/src/reference-test/**', '**/src/main/generated/**', '**/src/test/generated/**', '**/src/jmh/generated/**'
       removeUnusedImports()
       googleJavaFormat('1.10.0')
       importOrder 'org.hyperledger', 'java', ''

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/tracing/vm/VmTraceGenerator.java

@@ -288,7 +288,7 @@ private void initStep(final TraceFrame frame) {
     // set smart contract code
     if (currentTrace != null && "0x".equals(currentTrace.getCode())) {
       currentTrace.setCode(
-          currentTraceFrame.getMaybeCode().orElse(new Code()).getBytes().toHexString());
+          currentTraceFrame.getMaybeCode().orElse(Code.EMPTY_CODE).getBytes().toHexString());
     }
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java

@@ -36,7 +36,6 @@
 import org.hyperledger.besu.evm.Code;
 import org.hyperledger.besu.evm.Gas;
 import org.hyperledger.besu.evm.account.Account;
-import org.hyperledger.besu.evm.account.AccountState;
 import org.hyperledger.besu.evm.account.EvmAccount;
 import org.hyperledger.besu.evm.account.MutableAccount;
 import org.hyperledger.besu.evm.frame.MessageFrame;
@@ -366,13 +365,16 @@ public TransactionProcessingResult processTransaction(
         final Address contractAddress =
             Address.contractAddress(senderAddress, senderMutableAccount.getNonce() - 1L);
 
+        final Bytes initCodeBytes = transaction.getPayload();
         initialFrame =
             commonMessageFrameBuilder
                 .type(MessageFrame.Type.CONTRACT_CREATION)
                 .address(contractAddress)
                 .contract(contractAddress)
                 .inputData(Bytes.EMPTY)
-                .code(new Code(transaction.getPayload(), Hash.EMPTY))
+                .code(
+                    contractCreationProcessor.getCodeFromEVM(
+                        Hash.hash(initCodeBytes), initCodeBytes))
                 .build();
       } else {
         @SuppressWarnings("OptionalGetWithoutIsPresent") // isContractCall tests isPresent
@@ -385,9 +387,9 @@ public TransactionProcessingResult processTransaction(
                 .contract(to)
                 .inputData(transaction.getPayload())
                 .code(
-                    new Code(
-                        maybeContract.map(AccountState::getCode).orElse(Bytes.EMPTY),
-                        maybeContract.map(AccountState::getCodeHash).orElse(Hash.EMPTY)))
+                    maybeContract
+                        .map(c -> messageCallProcessor.getCodeFromEVM(c.getCodeHash(), c.getCode()))
+                        .orElse(Code.EMPTY_CODE))
                 .build();
       }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java

@@ -29,7 +29,6 @@
 import org.hyperledger.besu.evm.Code;
 import org.hyperledger.besu.evm.Gas;
 import org.hyperledger.besu.evm.account.Account;
-import org.hyperledger.besu.evm.account.AccountState;
 import org.hyperledger.besu.evm.account.EvmAccount;
 import org.hyperledger.besu.evm.account.MutableAccount;
 import org.hyperledger.besu.evm.frame.MessageFrame;
@@ -151,13 +150,16 @@ public TransactionProcessingResult processTransaction(
             previousNonce,
             privacyGroupId.toString());
 
+        final Bytes initCodeBytes = transaction.getPayload();
         initialFrame =
             commonMessageFrameBuilder
                 .type(MessageFrame.Type.CONTRACT_CREATION)
                 .address(privateContractAddress)
                 .contract(privateContractAddress)
                 .inputData(Bytes.EMPTY)
-                .code(new Code(transaction.getPayload(), Hash.EMPTY))
+                .code(
+                    contractCreationProcessor.getCodeFromEVM(
+                        Hash.hash(initCodeBytes), initCodeBytes))
                 .build();
       } else {
         final Address to = transaction.getTo().get();
@@ -170,9 +172,9 @@ public TransactionProcessingResult processTransaction(
                 .contract(to)
                 .inputData(transaction.getPayload())
                 .code(
-                    new Code(
-                        maybeContract.map(AccountState::getCode).orElse(Bytes.EMPTY),
-                        maybeContract.map(AccountState::getCodeHash).orElse(Hash.EMPTY)))
+                    maybeContract
+                        .map(c -> messageCallProcessor.getCodeFromEVM(c.getCodeHash(), c.getCode()))
+                        .orElse(Code.EMPTY_CODE))
                 .build();
       }
 

ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/MessageFrameTestFixture.java

@@ -17,7 +17,6 @@
 import static org.hyperledger.besu.evm.frame.MessageFrame.DEFAULT_MAX_STACK_SIZE;
 
 import org.hyperledger.besu.datatypes.Address;
-import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
@@ -52,7 +51,7 @@ public class MessageFrameTestFixture {
   private Wei gasPrice = Wei.ZERO;
   private Wei value = Wei.ZERO;
   private Bytes inputData = Bytes.EMPTY;
-  private Code code = new Code(Bytes.EMPTY, Hash.EMPTY);
+  private Code code = Code.EMPTY_CODE;
   private final List<UInt256> stackItems = new ArrayList<>();
   private Optional<BlockHeader> blockHeader = Optional.empty();
   private int depth = 0;

ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TestCodeExecutor.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.evm.Code;
+import org.hyperledger.besu.evm.EVM;
 import org.hyperledger.besu.evm.Gas;
 import org.hyperledger.besu.evm.account.MutableAccount;
 import org.hyperledger.besu.evm.frame.MessageFrame;
@@ -49,14 +50,19 @@ public TestCodeExecutor(final ProtocolSchedule protocolSchedule) {
   }
 
   public MessageFrame executeCode(
-      final String code, final long gasLimit, final Consumer<MutableAccount> accountSetup) {
+      final String codeHexString,
+      final long gasLimit,
+      final Consumer<MutableAccount> accountSetup) {
     final ProtocolSpec protocolSpec = fixture.getProtocolSchedule().getByBlockNumber(0);
     final WorldUpdater worldUpdater =
         createInitialWorldState(accountSetup, fixture.getStateArchive());
     final Deque<MessageFrame> messageFrameStack = new ArrayDeque<>();
 
+    final EVM evm = protocolSpec.getEvm();
     final MessageCallProcessor messageCallProcessor =
-        new MessageCallProcessor(protocolSpec.getEvm(), new PrecompileContractRegistry());
+        new MessageCallProcessor(evm, new PrecompileContractRegistry());
+    final Bytes codeBytes = Bytes.fromHexString(codeHexString);
+    final Code code = evm.getCode(Hash.hash(codeBytes), codeBytes);
 
     final Transaction transaction =
         Transaction.builder()
@@ -85,7 +91,7 @@ public MessageFrame executeCode(
             .inputData(transaction.getPayload())
             .sender(SENDER_ADDRESS)
             .value(transaction.getValue())
-            .code(new Code(Bytes.fromHexString(code), Hash.EMPTY))
+            .code(code)
             .blockHeader(blockHeader)
             .depth(0)
             .build();

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/EVMTest.java

@@ -50,7 +50,7 @@ public void setup() {
   @Test
   public void assertThatEndOfScriptNotExplicitlySetInCodeReturnsAVirtualOperation() {
     final Bytes noEnd = Bytes.fromHexString("0x60203560003555606035604035556000");
-    final Code code = new Code(noEnd, Hash.hash(noEnd));
+    final Code code = Code.createLegacyCode(noEnd, Hash.hash(noEnd));
     final Operation operation = evm.operationAtOffset(code, code.getSize());
     assertThat(operation).isNotNull();
     assertThat(operation.isVirtualOperation()).isTrue();
@@ -59,7 +59,7 @@ public void assertThatEndOfScriptNotExplicitlySetInCodeReturnsAVirtualOperation(
   @Test
   public void assertThatEndOfScriptExplicitlySetInCodeDoesNotReturnAVirtualOperation() {
     final Bytes ends = Bytes.fromHexString("0x6020356000355560603560403555600000");
-    final Code code = new Code(ends, Hash.hash(ends));
+    final Code code = Code.createLegacyCode(ends, Hash.hash(ends));
     when(operationRegistry.get(anyByte())).thenReturn(new StopOperation(gasCalculator));
     final Operation operation = evm.operationAtOffset(code, code.getSize() - 1);
     assertThat(operation).isNotNull();

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/operations/Create2OperationTest.java

@@ -25,6 +25,7 @@
 import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader;
 import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
 import org.hyperledger.besu.evm.Code;
+import org.hyperledger.besu.evm.EVM;
 import org.hyperledger.besu.evm.Gas;
 import org.hyperledger.besu.evm.account.MutableAccount;
 import org.hyperledger.besu.evm.frame.MessageFrame;
@@ -56,6 +57,7 @@ public class Create2OperationTest {
   private final WorldUpdater worldUpdater = mock(WorldUpdater.class);
   private final WrappedEvmAccount account = mock(WrappedEvmAccount.class);
   private final MutableAccount mutableAccount = mock(MutableAccount.class);
+  private final EVM evm = mock(EVM.class);
   private final Create2Operation operation =
       new Create2Operation(new ConstantinopleGasCalculator());
 
@@ -141,7 +143,7 @@ public void setUp() {
             .sender(Address.fromHexString(sender))
             .value(Wei.ZERO)
             .apparentValue(Wei.ZERO)
-            .code(new Code(codeBytes, Hash.hash(codeBytes)))
+            .code(Code.createLegacyCode(codeBytes, Hash.hash(codeBytes)))
             .depth(1)
             .completer(__ -> {})
             .address(Address.fromHexString(sender))
@@ -164,6 +166,10 @@ public void setUp() {
     when(mutableAccount.getBalance()).thenReturn(Wei.ZERO);
     when(worldUpdater.getAccount(any())).thenReturn(account);
     when(worldUpdater.updater()).thenReturn(worldUpdater);
+    when(evm.getCode(any(), any()))
+        .thenAnswer(
+            invocation ->
+                Code.createLegacyCode(invocation.getArgument(1), invocation.getArgument(0)));
   }
 
   @Test
@@ -174,7 +180,7 @@ public void shouldCalculateAddress() {
 
   @Test
   public void shouldCalculateGasPrice() {
-    final OperationResult result = operation.execute(messageFrame, null);
+    final OperationResult result = operation.execute(messageFrame, evm);
     assertThat(result.getHaltReason()).isEmpty();
     assertThat(result.getGasCost()).contains(Gas.of(expectedGas));
   }

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/operations/CreateOperationTest.java

@@ -88,7 +88,7 @@ public void createFromMemoryMutationSafe() {
             .sender(Address.fromHexString(SENDER))
             .value(Wei.ZERO)
             .apparentValue(Wei.ZERO)
-            .code(new Code(SIMPLE_CREATE, Hash.hash(SIMPLE_CREATE)))
+            .code(Code.createLegacyCode(SIMPLE_CREATE, Hash.hash(SIMPLE_CREATE)))
             .depth(1)
             .completer(__ -> {})
             .address(Address.fromHexString(SENDER))

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/operations/JumpOperationTest.java

@@ -96,7 +96,7 @@ public void shouldJumpWhenLocationIsJumpDest() {
     final MessageFrame frame =
         createMessageFrameBuilder(Gas.of(10_000))
             .pushStackItem(UInt256.fromHexString("0x03"))
-            .code(new Code(jumpBytes, Hash.hash(jumpBytes)))
+            .code(Code.createLegacyCode(jumpBytes, Hash.hash(jumpBytes)))
             .build();
     frame.setPC(CURRENT_PC);
 
@@ -111,7 +111,7 @@ public void shouldJumpWhenLocationIsJumpDestAndAtEndOfCode() {
     final MessageFrame frame =
         createMessageFrameBuilder(Gas.of(10_000))
             .pushStackItem(UInt256.fromHexString("0x03"))
-            .code(new Code(jumpBytes, Hash.hash(jumpBytes)))
+            .code(Code.createLegacyCode(jumpBytes, Hash.hash(jumpBytes)))
             .build();
     frame.setPC(CURRENT_PC);
 
@@ -126,7 +126,7 @@ public void shouldHaltWithInvalidJumDestinationWhenLocationIsOutsideOfCodeRange(
     final MessageFrame frameDestinationGreaterThanCodeSize =
         createMessageFrameBuilder(Gas.of(100))
             .pushStackItem(UInt256.fromHexString("0xFFFFFFFF"))
-            .code(new Code(jumpBytes, Hash.hash(jumpBytes)))
+            .code(Code.createLegacyCode(jumpBytes, Hash.hash(jumpBytes)))
             .build();
     frameDestinationGreaterThanCodeSize.setPC(CURRENT_PC);
 
@@ -136,7 +136,7 @@ public void shouldHaltWithInvalidJumDestinationWhenLocationIsOutsideOfCodeRange(
     final MessageFrame frameDestinationEqualsToCodeSize =
         createMessageFrameBuilder(Gas.of(100))
             .pushStackItem(UInt256.fromHexString("0x04"))
-            .code(new Code(badJump, Hash.hash(badJump)))
+            .code(Code.createLegacyCode(badJump, Hash.hash(badJump)))
             .build();
     frameDestinationEqualsToCodeSize.setPC(CURRENT_PC);
 
@@ -154,7 +154,7 @@ public void longContractsValidate() {
     final MessageFrame longContract =
         createMessageFrameBuilder(Gas.of(100))
             .pushStackItem(UInt256.fromHexString("0x12c"))
-            .code(new Code(longCode, Hash.hash(longCode)))
+            .code(Code.createLegacyCode(longCode, Hash.hash(longCode)))
             .build();
     longContract.setPC(255);
 
@@ -166,7 +166,7 @@ public void longContractsValidate() {
   public void shouldReuseJumpDestMap() {
     final JumpOperation operation = new JumpOperation(gasCalculator);
     final Bytes jumpBytes = Bytes.fromHexString("0x6003565b00");
-    Code getsCached = spy(new Code(jumpBytes, Hash.hash(jumpBytes)));
+    Code getsCached = spy(Code.createLegacyCode(jumpBytes, Hash.hash(jumpBytes)));
     MessageFrame frame =
         createMessageFrameBuilder(Gas.of(10_000))
             .pushStackItem(UInt256.fromHexString("0x03"))

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java

@@ -213,6 +213,7 @@ public void run() {
       final PrecompileContractRegistry precompileContractRegistry =
           protocolSpec.getPrecompileContractRegistry();
       final EVM evm = protocolSpec.getEvm();
+      Code code = evm.getCode(Hash.hash(codeHexString), codeHexString);
       final Stopwatch stopwatch = Stopwatch.createUnstarted();
       long lastTime = 0;
       do {
@@ -242,7 +243,7 @@ public void run() {
                 .inputData(callData)
                 .value(ethValue)
                 .apparentValue(ethValue)
-                .code(new Code(codeHexString, Hash.hash(codeHexString)))
+                .code(code)
                 .blockValues(blockHeader)
                 .depth(0)
                 .completer(c -> {})

evm/src/main/java/org/hyperledger/besu/evm/Code.java

@@ -19,11 +19,14 @@
 import org.hyperledger.besu.evm.operation.PushOperation;
 
 import com.google.common.base.MoreObjects;
+import com.google.errorprone.annotations.RestrictedApi;
 import org.apache.tuweni.bytes.Bytes;
 
 /** Represents EVM code associated with an account. */
 public class Code {
 
+  public static final Code EMPTY_CODE = new Code(Bytes.EMPTY, Hash.EMPTY);
+
   /** The bytes representing the code. */
   private final Bytes bytes;
 
@@ -42,19 +45,17 @@ public class Code {
    * @param bytes The byte representation of the code.
    * @param codeHash the Hash of the bytes in the code.
    */
-  public Code(final Bytes bytes, final Hash codeHash) {
+  protected Code(final Bytes bytes, final Hash codeHash) {
     this.bytes = bytes;
     this.codeHash = codeHash;
   }
 
-  public Code(final Bytes bytecode, final Hash codeHash, final long[] validJumpDestinations) {
-    this.bytes = bytecode;
-    this.validJumpDestinations = validJumpDestinations;
-    this.codeHash = codeHash;
-  }
-
-  public Code() {
-    this(Bytes.EMPTY, Hash.EMPTY);
+  @RestrictedApi(
+      explanation = "To be used for testing purpose only",
+      link = "",
+      allowedOnPath = ".*/src/test/.*")
+  public static Code createLegacyCode(final Bytes bytes, final Hash codeHash) {
+    return new Code(bytes, codeHash);
   }
 
   /**
@@ -88,24 +89,24 @@ public int getSize() {
   }
 
   public long[] calculateJumpDests() {
-    int size = getSize();
-    long[] bitmap = new long[(size >> 6) + 1];
-    byte[] rawCode = getBytes().toArrayUnsafe();
-    int length = rawCode.length;
+    final int size = getSize();
+    final long[] bitmap = new long[(size >> 6) + 1];
+    final byte[] rawCode = getBytes().toArrayUnsafe();
+    final int length = rawCode.length;
     for (int i = 0; i < length; ) {
       long thisEntry = 0L;
-      int entryPos = i >> 6;
-      int max = Math.min(64, length - (entryPos << 6));
+      final int entryPos = i >> 6;
+      final int max = Math.min(64, length - (entryPos << 6));
       int j = i & 0x3F;
       for (; j < max; i++, j++) {
-        byte operationNum = rawCode[i];
+        final byte operationNum = rawCode[i];
         if (operationNum == JumpDestOperation.OPCODE) {
           thisEntry |= 1L << j;
         } else if (operationNum > PushOperation.PUSH_BASE) {
           // not needed - && operationNum <= PushOperation.PUSH_MAX
           // Java quirk, all bytes are signed, and PUSH32 is 127, which is Byte.MAX_VALUE
           // so we don't need to check the upper bound as it will never be violated
-          int multiByteDataLen = operationNum - PushOperation.PUSH_BASE;
+          final int multiByteDataLen = operationNum - PushOperation.PUSH_BASE;
           j += multiByteDataLen;
           i += multiByteDataLen;
         }
@@ -129,7 +130,16 @@ public Hash getCodeHash() {
     return codeHash;
   }
 
-  public long[] getValidJumpDestinations() {
-    return validJumpDestinations;
+  public boolean isJumpDestInvalid(final int jumpDestination) {
+    if (jumpDestination < 0 || jumpDestination >= getSize()) {
+      return true;
+    }
+    if (validJumpDestinations == null || validJumpDestinations.length == 0) {
+      validJumpDestinations = calculateJumpDests();
+    }
+
+    final long targetLong = validJumpDestinations[jumpDestination >>> 6];
+    final long targetBit = 1L << (jumpDestination & 0x3F);
+    return (targetLong & targetBit) == 0L;
   }
 }