Skip to content

Commit

Permalink
Merge branch 'develop' into 14293-skip-codacy-checks
Browse files Browse the repository at this point in the history
  • Loading branch information
@rbarkerSL
rbarkerSL committed Sep 23, 2024
2 parents d98402d + 8dfd416 commit 9ac0dfb
Show file tree
Hide file tree
Showing 82 changed files with 1,326 additions and 2,977 deletions.
4 changes: 2 additions & 2 deletions hedera-dependency-versions/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ dependencies.constraints {
api("com.google.jimfs:jimfs:1.2") {
because("com.google.jimfs")
}
api("com.google.protobuf:protobuf-java:3.25.4") {
api("com.google.protobuf:protobuf-java:4.28.2") {
because("com.google.protobuf")
}
api("com.google.protobuf:protobuf-java-util:3.25.4") {
api("com.google.protobuf:protobuf-java-util:4.28.2") {
because("com.google.protobuf.util")
}
api("com.hedera.pbj:pbj-runtime:0.9.2") {
Expand Down
2 changes: 1 addition & 1 deletion ...ode/hedera-app/src/main/java/com/hedera/node/app/workflows/handle/record/SystemSetup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ public void doPostUpgradeSetup(@NonNull final Dispatch dispatch) {

// We update the node details file from the address book that resulted from all pre-upgrade HAPI node changes
final var nodeStore = dispatch.handleContext().storeFactory().readableStore(ReadableNodeStore.class);
fileService.updateNodeDetailsAfterFreeze(systemContext, nodeStore);
fileService.updateAddressBookAndNodeDetailsAfterFreeze(systemContext, nodeStore);
dispatch.stack().commitFullStack();

// And then we update the system files for fees schedules, throttles, override properties, and override
Expand Down
3 changes: 1 addition & 2 deletions hedera-node/hedera-app/src/main/java/com/hedera/node/app/workflows/handle/steps/UserTxn.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,13 +118,12 @@ public static UserTxn from(
} else {
type = ORDINARY_TRANSACTION;
}
final var isGenesis = lastHandledConsensusTime.equals(Instant.EPOCH);
final var config = configProvider.getConfiguration();
final var consensusConfig = config.getConfigData(ConsensusConfig.class);
final var blockStreamConfig = config.getConfigData(BlockStreamConfig.class);
final var stack = SavepointStackImpl.newRootStack(
state,
isGenesis ? Integer.MAX_VALUE : consensusConfig.handleMaxPrecedingRecords(),
type != ORDINARY_TRANSACTION ? Integer.MAX_VALUE : consensusConfig.handleMaxPrecedingRecords(),
consensusConfig.handleMaxFollowingRecords(),
boundaryStateChangeListener,
kvStateChangeListener,
Expand Down
10 changes: 5 additions & 5 deletions .../hedera-app/src/test/java/com/hedera/node/app/workflows/handle/steps/SystemSetupTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ void successfulAutoUpdatesAreDispatchedWithFilesAvailable() throws IOException {
subject.doPostUpgradeSetup(dispatch);

final var filesConfig = config.getConfigData(FilesConfig.class);
verify(fileService).updateNodeDetailsAfterFreeze(any(SystemContext.class), eq(readableNodeStore));
verify(fileService).updateAddressBookAndNodeDetailsAfterFreeze(any(SystemContext.class), eq(readableNodeStore));
verifyUpdateDispatch(filesConfig.networkProperties(), serializedPropertyOverrides());
verifyUpdateDispatch(filesConfig.hapiPermissions(), serializedPermissionOverrides());
verifyUpdateDispatch(filesConfig.throttleDefinitions(), serializedThrottleOverrides());
Expand All @@ -186,7 +186,7 @@ void successfulAutoUpdatesAreDispatchedWithFilesAvailable() throws IOException {
}

@Test
void onlyNodeDetailsAutoUpdateIsDispatchedWithNoFilesAvailable() {
void onlyAddressBookAndNodeDetailsAutoUpdateIsDispatchedWithNoFilesAvailable() {
final var config = HederaTestConfigBuilder.create()
.withValue("networkAdmin.upgradeSysFilesLoc", tempDir.toString())
.getOrCreateConfig();
Expand All @@ -198,7 +198,7 @@ void onlyNodeDetailsAutoUpdateIsDispatchedWithNoFilesAvailable() {

subject.doPostUpgradeSetup(dispatch);

verify(fileService).updateNodeDetailsAfterFreeze(any(SystemContext.class), eq(readableNodeStore));
verify(fileService).updateAddressBookAndNodeDetailsAfterFreeze(any(SystemContext.class), eq(readableNodeStore));
verify(stack, times(1)).commitFullStack();

final var infoLogs = logCaptor.infoLogs();
Expand All @@ -210,7 +210,7 @@ void onlyNodeDetailsAutoUpdateIsDispatchedWithNoFilesAvailable() {
}

@Test
void onlyNodeDetailsAutoUpdateIsDispatchedWithInvalidFilesAvailable() throws IOException {
void onlyAddressBookAndNodeDetailsAutoUpdateIsDispatchedWithInvalidFilesAvailable() throws IOException {
final var config = HederaTestConfigBuilder.create()
.withValue("networkAdmin.upgradeSysFilesLoc", tempDir.toString())
.getOrCreateConfig();
Expand All @@ -227,7 +227,7 @@ void onlyNodeDetailsAutoUpdateIsDispatchedWithInvalidFilesAvailable() throws IOE

subject.doPostUpgradeSetup(dispatch);

verify(fileService).updateNodeDetailsAfterFreeze(any(SystemContext.class), eq(readableNodeStore));
verify(fileService).updateAddressBookAndNodeDetailsAfterFreeze(any(SystemContext.class), eq(readableNodeStore));
verify(stack, times(1)).commitFullStack();

final var errorLogs = logCaptor.errorLogs();
Expand Down
6 changes: 3 additions & 3 deletions ...ile-service-impl/src/main/java/com/hedera/node/app/service/file/impl/FileServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,13 @@ public V0490FileSchema fileSchema() {
}

/**
* Creates the 102 files in the given genesis context.
* Update the 101, 102 files with the nodeStore data.
*
* @param context the genesis context
* @param nodeStore the ReadableNodeStore
*/
public void updateNodeDetailsAfterFreeze(
public void updateAddressBookAndNodeDetailsAfterFreeze(
@NonNull final SystemContext context, @NonNull final ReadableNodeStore nodeStore) {
fileSchema.updateNodeDetailsAfterFreeze(context, nodeStore);
fileSchema.updateAddressBookAndNodeDetailsAfterFreeze(context, nodeStore);
}
}
45 changes: 31 additions & 14 deletions ...ice-impl/src/main/java/com/hedera/node/app/service/file/impl/schemas/V0490FileSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,14 +235,17 @@ public Bytes genesisNodeDetails(@NonNull final NetworkInfo networkInfo) {
NodeAddressBook.newBuilder().nodeAddress(nodeDetails).build());
}

public void updateNodeDetailsAfterFreeze(
public void updateAddressBookAndNodeDetailsAfterFreeze(
@NonNull final SystemContext systemContext, @NonNull final ReadableNodeStore nodeStore) {
requireNonNull(systemContext);
final var config = systemContext.configuration();
final var filesConfig = config.getConfigData(FilesConfig.class);
// Create the node details for file 102
// Create the nodeDetails for file 102
dispatchSynthFileUpdate(
systemContext, createFileID(filesConfig.nodeDetails(), config), nodeStoreNodeDetails(nodeStore));
// Create the addressBook for file 101
dispatchSynthFileUpdate(
systemContext, createFileID(filesConfig.addressBook(), config), nodeStoreAddressBook(nodeStore));
}

/**
Expand Down Expand Up @@ -281,22 +284,36 @@ private Bytes nodeStoreNodeDetails(@NonNull final ReadableNodeStore nodeStore) {
.mapToLong(EntityNumber::number)
.mapToObj(nodeStore::get)
.filter(node -> node != null && !node.deleted())
.forEach(node -> {
nodeDetails.add(NodeAddress.newBuilder()
.nodeId(node.nodeId())
.nodeAccountId(node.accountId())
.nodeCertHash(node.grpcCertificateHash())
.description(node.description())
.stake(node.weight())
.rsaPubKey(readableKey(getPublicKeyFromCertBytes(
node.gossipCaCertificate().toByteArray(), node.nodeId())))
.serviceEndpoint(node.serviceEndpoint())
.build());
});
.forEach(node -> nodeDetails.add(NodeAddress.newBuilder()
.nodeId(node.nodeId())
.nodeAccountId(node.accountId())
.nodeCertHash(node.grpcCertificateHash())
.description(node.description())
.stake(node.weight())
.rsaPubKey(readableKey(getPublicKeyFromCertBytes(
node.gossipCaCertificate().toByteArray(), node.nodeId())))
.serviceEndpoint(node.serviceEndpoint())
.build()));
return NodeAddressBook.PROTOBUF.toBytes(
NodeAddressBook.newBuilder().nodeAddress(nodeDetails).build());
}

private Bytes nodeStoreAddressBook(@NonNull final ReadableNodeStore nodeStore) {
final var nodeAddresses = new ArrayList<NodeAddress>();
StreamSupport.stream(Spliterators.spliterator(nodeStore.keys(), nodeStore.sizeOfState(), DISTINCT), false)
.mapToLong(EntityNumber::number)
.mapToObj(nodeStore::get)
.filter(node -> node != null && !node.deleted())
.forEach(node -> nodeAddresses.add(NodeAddress.newBuilder()
.nodeId(node.nodeId())
.nodeCertHash(node.grpcCertificateHash())
.nodeAccountId(node.accountId())
.serviceEndpoint(node.serviceEndpoint())
.build()));
return NodeAddressBook.PROTOBUF.toBytes(
NodeAddressBook.newBuilder().nodeAddress(nodeAddresses).build());
}

// ================================================================================================================
// Creates and loads the initial Fee Schedule into state

Expand Down
84 changes: 77 additions & 7 deletions ...st-clients/src/main/java/com/hedera/services/bdd/suites/hip993/SystemFileExportsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,33 @@ final Stream<DynamicTest> syntheticNodeDetailsUpdateHappensAtUpgradeBoundary() {
final AtomicReference<Map<Long, X509Certificate>> gossipCertificates = new AtomicReference<>();
return hapiTest(
recordStreamMustIncludePassFrom(selectedItems(
addressBookExportValidator(grpcCertHashes, gossipCertificates), 1, this::isSysFileUpdate)),
nodeDetailsExportValidator(grpcCertHashes, gossipCertificates), 1, this::isSysFileUpdate)),
given(() -> gossipCertificates.set(generateCertificates(CLASSIC_HAPI_TEST_NETWORK_SIZE))),
// This is the genesis transaction
cryptoCreate("firstUser"),
overriding("nodes.updateAccountIdAllowed", "true"),
sourcing(() -> blockingOrder(nOps(CLASSIC_HAPI_TEST_NETWORK_SIZE, i -> nodeUpdate("" + i)
.accountId("0.0." + (i + ACCOUNT_ID_OFFSET))
.description(DESCRIPTION_PREFIX + i)
.serviceEndpoint(endpointsFor(i))
.grpcCertificateHash(grpcCertHashes[i])
.gossipCaCertificate(derEncoded(gossipCertificates.get().get((long) i)))))),
// And now simulate an upgrade boundary
simulatePostUpgradeTransaction(),
cryptoCreate("secondUser").via("addressBookExport"));
}

@GenesisHapiTest
final Stream<DynamicTest> syntheticAddressBookUpdateHappensAtUpgradeBoundary() {
final var grpcCertHashes = new byte[][] {
randomUtf8Bytes(48), randomUtf8Bytes(48), randomUtf8Bytes(48), randomUtf8Bytes(48),
};
final AtomicReference<Map<Long, X509Certificate>> gossipCertificates = new AtomicReference<>();
return hapiTest(
recordStreamMustIncludePassFrom(selectedItems(
addressBookExportValidator("files.addressBook", grpcCertHashes, gossipCertificates),
2,
this::isSysFileUpdate)),
given(() -> gossipCertificates.set(generateCertificates(CLASSIC_HAPI_TEST_NETWORK_SIZE))),
// This is the genesis transaction
cryptoCreate("firstUser"),
Expand All @@ -151,7 +177,7 @@ final Stream<DynamicTest> syntheticFeeSchedulesUpdateHappensAtUpgradeBoundary()
recordStreamMustIncludePassFrom(selectedItems(
sysFileExportValidator(
"files.feeSchedules", upgradeFeeSchedules, SystemFileExportsTest::parseFeeSchedule),
2,
3,
this::isSysFileUpdate)),
// This is the genesis transaction
sourcingContextual(spec -> overridingTwo(
Expand Down Expand Up @@ -196,7 +222,7 @@ final Stream<DynamicTest> syntheticThrottlesUpdateHappensAtUpgradeBoundary() thr
"files.throttleDefinitions",
upgradeThrottleDefs,
SystemFileExportsTest::parseThrottleDefs),
2,
3,
this::isSysFileUpdate)),
// This is the genesis transaction
sourcingContextual(spec -> overridingTwo(
Expand Down Expand Up @@ -237,7 +263,7 @@ final Stream<DynamicTest> syntheticPropertyOverridesUpdateHappensAtUpgradeBounda
"files.networkProperties",
upgradePropOverrides,
SystemFileExportsTest::parseConfigList),
2,
3,
this::isSysFileUpdate)),
// This is the genesis transaction
sourcingContextual(spec -> overriding(
Expand Down Expand Up @@ -272,7 +298,7 @@ final Stream<DynamicTest> syntheticPropertyOverridesUpdateCanBeEmptyFile() {
"files.networkProperties",
ServicesConfigurationList.getDefaultInstance(),
SystemFileExportsTest::parseConfigList),
2,
3,
this::isSysFileUpdate)),
// This is the genesis transaction
sourcingContextual(spec -> overridingTwo(
Expand Down Expand Up @@ -313,7 +339,7 @@ final Stream<DynamicTest> syntheticPermissionOverridesUpdateHappensAtUpgradeBoun
"files.hapiPermissions",
upgradePermissionOverrides,
SystemFileExportsTest::parseConfigList),
2,
3,
this::isSysFileUpdate)),
// This is the genesis transaction
sourcingContextual(spec -> overriding(
Expand Down Expand Up @@ -401,7 +427,7 @@ private static <T> VisibleItemsValidator sysFileExportValidator(
};
}

private static VisibleItemsValidator addressBookExportValidator(
private static VisibleItemsValidator nodeDetailsExportValidator(
@NonNull final byte[][] grpcCertHashes,
@NonNull final AtomicReference<Map<Long, X509Certificate>> gossipCertificates) {
return (spec, records) -> {
Expand Down Expand Up @@ -448,6 +474,50 @@ private static VisibleItemsValidator addressBookExportValidator(
};
}

private static VisibleItemsValidator addressBookExportValidator(
@NonNull final String fileNumProperty,
@NonNull final byte[][] grpcCertHashes,
@NonNull final AtomicReference<Map<Long, X509Certificate>> gossipCertificates) {
return (spec, records) -> {
final var items = records.get(SELECTED_ITEMS_KEY);
assertNotNull(items, "No post-upgrade txn found");
final var targetId =
new FileID(0, 0, Long.parseLong(spec.startupProperties().get(fileNumProperty)));
final var updateItem = items.entries().stream()
.filter(item -> item.function() == FileUpdate)
.filter(item ->
toPbj(item.body().getFileUpdate().getFileID()).equals(targetId))
.findFirst()
.orElse(null);
assertNotNull(updateItem, "No update for " + fileNumProperty + " found in post-upgrade txn");
final var synthOp = updateItem.body().getFileUpdate();
final var addressBookId =
new FileID(0, 0, Long.parseLong(spec.startupProperties().get("files.addressBook")));
assertEquals(addressBookId, toPbj(synthOp.getFileID()));
try {
final var updatedAddressBook = NodeAddressBook.PROTOBUF.parse(
Bytes.wrap(synthOp.getContents().toByteArray()));
for (final var address : updatedAddressBook.nodeAddress()) {
final var actualCertHash = address.nodeCertHash().toByteArray();
assertArrayEquals(
grpcCertHashes[(int) address.nodeId()],
actualCertHash,
"node" + address.nodeId() + " has wrong cert hash");

final var expectedAccountID = AccountID.newBuilder()
.accountNum(address.nodeId() + ACCOUNT_ID_OFFSET)
.build();
assertEquals(expectedAccountID, address.nodeAccountId());

final var expectedServiceEndpoint = endpointsFor((int) address.nodeId());
assertEquals(expectedServiceEndpoint, address.serviceEndpoint());
}
} catch (ParseException e) {
Assertions.fail("Update contents was not protobuf " + e.getMessage());
}
};
}

private static VisibleItemsValidator validatorFor(
@NonNull final AtomicReference<Map<FileID, Bytes>> preGenesisContents) {
return (spec, records) -> validateSystemFileExports(spec, records, preGenesisContents.get());
Expand Down
Loading

0 comments on commit 9ac0dfb

Please sign in to comment.