feat: update state validation tool to be compatible with the Mega Map

hedera-state-validator/src/main/java/com/hedera/statevalidation/Constants.java

@@ -8,4 +8,5 @@ public class Constants {
     public static final Path COMBINED_REPORT_FILE = Path.of("combined_report.json");
     public static final String COMBINED_HTML_REPORT_TEMPLATE = "combined_report.html.template";
     public static final Path COMBINED_HTML_REPORT = Path.of("combined_report.html");
+    public static final String VM_LABEL = "state";
 }

hedera-state-validator/src/main/java/com/hedera/statevalidation/parameterresolver/InitUtils.java

@@ -2,6 +2,7 @@
 package com.hedera.statevalidation.parameterresolver;
 
 import static com.hedera.node.app.spi.fees.NoopFeeCharging.NOOP_FEE_CHARGING;
+import static com.hedera.statevalidation.Constants.VM_LABEL;
 import static com.hedera.statevalidation.parameterresolver.StateResolver.readVersion;
 import static com.hedera.statevalidation.validators.Constants.FILE_CHANNELS;
 import static com.hedera.statevalidation.validators.Constants.STATE_DIR;
@@ -44,8 +45,6 @@
 import com.hedera.node.app.signature.impl.SignatureVerifierImpl;
 import com.hedera.node.app.spi.AppContext;
 import com.hedera.node.app.spi.fixtures.info.FakeNetworkInfo;
-import com.hedera.node.app.state.merkle.MerkleSchemaRegistry;
-import com.hedera.node.app.state.merkle.SchemaApplications;
 import com.hedera.node.app.state.recordcache.RecordCacheService;
 import com.hedera.node.app.throttle.AppThrottleFactory;
 import com.hedera.node.app.throttle.CongestionThrottleService;
@@ -98,44 +97,21 @@
 import com.swirlds.platform.state.service.PlatformStateFacade;
 import com.swirlds.platform.state.service.PlatformStateService;
 import com.swirlds.state.State;
-import com.swirlds.state.lifecycle.Schema;
-import com.swirlds.state.lifecycle.SchemaRegistry;
-import com.swirlds.state.lifecycle.StateMetadata;
-import com.swirlds.state.merkle.disk.OnDiskKeySerializer;
-import com.swirlds.state.merkle.disk.OnDiskValueSerializer;
 import com.swirlds.virtualmap.VirtualMap;
 import com.swirlds.virtualmap.config.VirtualMapConfig;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.InstantSource;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.hiero.base.constructable.ConstructableRegistry;
 import org.hiero.base.crypto.config.CryptoConfig;
 
 public class InitUtils {
 
-    private static final Logger log = LogManager.getLogger(InitUtils.class);
-
-    /**
-     * The excluded tables were renamed (see https://github.com/hashgraph/hedera-services/pull/16775). However, their metadata
-     * still remains till the next version. This confuses the validator as it expects these tables to exist while they don't.
-     * Hence, we exclude them manually.
-     */
-    private static final Set<String> TABLES_TO_EXCLUDE = Set.of(
-            "ScheduleService.SCHEDULES_BY_EQUALITY",
-            "ScheduleService.SCHEDULES_BY_EXPIRY_SEC",
-            "HintsService.PREPROCESSING_VOTES",
-            "HintsService.HINTS_KEY_SETS",
-            "HintsService.CRS_PUBLICATIONS");
-
     public static Configuration CONFIGURATION;
 
     /**
@@ -185,70 +161,20 @@ public static Configuration getConfiguration() {
     }
 
     /**
-     * This method initializes all the virtual maps and their data sources
+     * This method initializes the virtual map and its data source
      *
-     * @param servicesRegistry the services registry required to build VMs
-     * @return the list of virtual maps and their data sources
+     * @return the virtual map and its data source
      */
-    static List<VirtualMapAndDataSourceRecord<?, ?>> initVirtualMapRecords(ServicesRegistryImpl servicesRegistry) {
+    static VirtualMapAndDataSourceRecord initVirtualMapRecord() {
         final Path stateDirPath = Paths.get(STATE_DIR);
-
         final MerkleDb merkleDb = MerkleDb.getInstance(stateDirPath, CONFIGURATION);
         Map<String, MerkleDbTableConfig> tableConfigByNames = merkleDb.getTableConfigs();
-        final var virtualMaps = new ArrayList<VirtualMapAndDataSourceRecord<?, ?>>();
-
-        servicesRegistry.registrations().forEach((registration) -> {
-            try {
-                var service = registration.service();
-                var serviceName = service.getServiceName();
-                log.debug("Registering schemas for service {}", serviceName);
-                var registry =
-                        new MerkleSchemaRegistry(
-                                ConstructableRegistry.getInstance(),
-                                serviceName,
-                                CONFIGURATION,
-                                new SchemaApplications()) {
-                            @SuppressWarnings({"rawtypes", "unchecked"})
-                            @Override
-                            public SchemaRegistry register(Schema schema) {
-                                schema.statesToCreate().forEach((def) -> {
-                                    if (!def.onDisk()) {
-                                        return;
-                                    }
-                                    final var md = new StateMetadata<>(serviceName, schema, def);
-                                    final var label = StateMetadata.computeLabel(serviceName, def.stateKey());
-                                    if (TABLES_TO_EXCLUDE.contains(label)) {
-                                        return;
-                                    }
-                                    MerkleDbTableConfig tableConfig = tableConfigByNames.get(label);
-                                    final var keySerializer = new OnDiskKeySerializer<>(
-                                            md.onDiskKeySerializerClassId(),
-                                            md.onDiskKeyClassId(),
-                                            md.stateDefinition().keyCodec());
-                                    final var valueSerializer = new OnDiskValueSerializer<>(
-                                            md.onDiskValueSerializerClassId(),
-                                            md.onDiskValueClassId(),
-                                            md.stateDefinition().valueCodec());
-                                    final var ds = new RestoringMerkleDbDataSourceBuilder<>(stateDirPath, tableConfig);
-                                    final var vm = new VirtualMap(label, ds, CONFIGURATION);
-                                    virtualMaps.add(new VirtualMapAndDataSourceRecord<>(
-                                            label,
-                                            (MerkleDbDataSource) vm.getDataSource(),
-                                            vm,
-                                            keySerializer,
-                                            valueSerializer));
-                                });
-                                return null;
-                            }
-                        };
+        MerkleDbTableConfig tableConfig = tableConfigByNames.get(VM_LABEL);
 
-                service.registerSchemas(registry);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
+        final var dataSourceBuilder = new RestoringMerkleDbDataSourceBuilder<>(stateDirPath, tableConfig);
+        final var virtualMap = new VirtualMap(VM_LABEL, dataSourceBuilder, CONFIGURATION);
 
-        return virtualMaps;
+        return new VirtualMapAndDataSourceRecord(VM_LABEL, (MerkleDbDataSource) virtualMap.getDataSource(), virtualMap);
     }
 
     /**
@@ -328,25 +254,20 @@ static ServicesRegistryImpl initServiceRegistry() {
      * This method initializes the State API
      */
     static void initServiceMigrator(State state, Configuration configuration, ServicesRegistry servicesRegistry) {
-        final var bootstrapConfigProvider = new BootstrapConfigProviderImpl();
         final var serviceMigrator = new OrderedServiceMigrator();
         final var platformFacade = PlatformStateFacade.DEFAULT_PLATFORM_STATE_FACADE;
         final var deserializedVersion = platformFacade.creationSoftwareVersionOf(state);
-        final var version = getNodeStartupVersion(bootstrapConfigProvider.getConfiguration());
+        final var nodeStartupVersion = readVersion();
         serviceMigrator.doMigrations(
                 (MerkleNodeState) state,
                 servicesRegistry,
                 deserializedVersion,
-                version,
+                nodeStartupVersion,
                 configuration,
                 configuration,
                 new FakeStartupNetworks(Network.newBuilder().build()),
                 new StoreMetricsServiceImpl(new NoOpMetrics()),
                 new ConfigProviderImpl(),
                 platformFacade);
     }
-
-    private static SemanticVersion getNodeStartupVersion(final Configuration config) {
-        return readVersion();
-    }
 }

hedera-state-validator/src/main/java/com/hedera/statevalidation/parameterresolver/StateResolver.java

@@ -8,7 +8,6 @@
 import static com.swirlds.platform.state.snapshot.SignedStateFileReader.readStateFile;
 
 import com.hedera.hapi.node.base.SemanticVersion;
-import com.hedera.node.app.HederaStateRoot;
 import com.hedera.node.app.HederaVirtualMapState;
 import com.hedera.node.app.roster.RosterService;
 import com.hedera.node.app.services.ServicesRegistryImpl;
@@ -30,7 +29,6 @@
 import com.swirlds.platform.state.snapshot.DeserializedSignedState;
 import com.swirlds.platform.util.BootstrapUtils;
 import com.swirlds.state.State;
-import com.swirlds.state.merkle.MerkleStateRoot;
 import com.swirlds.virtualmap.constructable.ConstructableUtils;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -40,7 +38,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.hiero.base.concurrent.ExecutorFactory;
-import org.hiero.base.constructable.ClassConstructorPair;
 import org.hiero.base.constructable.ConstructableRegistry;
 import org.hiero.base.constructable.ConstructableRegistryException;
 import org.hiero.base.crypto.config.CryptoConfig;
@@ -70,12 +67,12 @@ public DeserializedSignedState resolveParameter(
 
         if (deserializedSignedState == null) {
             try {
-
                 initState();
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
         }
+
         return deserializedSignedState;
     }
 
@@ -91,10 +88,8 @@ public static DeserializedSignedState initState() throws IOException {
                 HederaVirtualMapState::new,
                 platformStateFacade,
                 platformContext);
-        final MerkleStateRoot servicesState = (MerkleStateRoot)
-                deserializedSignedState.reservedSignedState().get().getState();
 
-        initServiceMigrator(servicesState, platformContext.getConfiguration(), serviceRegistry);
+        initServiceMigrator(getState(), platformContext.getConfiguration(), serviceRegistry);
 
         return deserializedSignedState;
     }
@@ -138,10 +133,7 @@ private static PlatformContext createPlatformContext() {
 
             ConstructableUtils.registerVirtualMapConstructables(getConfiguration());
             BootstrapUtils.setupConstructableRegistryWithConfiguration(getConfiguration());
-            final SemanticVersion servicesVersion = readVersion();
 
-            ConstructableRegistry.getInstance()
-                    .registerConstructable(new ClassConstructorPair(MerkleStateRoot.class, HederaStateRoot::new));
         } catch (ConstructableRegistryException e) {
             throw new RuntimeException(e);
         }

hedera-state-validator/src/main/java/com/hedera/statevalidation/parameterresolver/VirtualMapAndDataSourceProvider.java

@@ -9,6 +9,6 @@
 public class VirtualMapAndDataSourceProvider implements ArgumentsProvider {
     @Override
     public Stream<? extends Arguments> provideArguments(ExtensionContext context) {
-        return VirtualMapHolder.getInstance().getRecords().stream().map(Arguments::of);
+        return Stream.of(Arguments.of(VirtualMapHolder.getInstance().getRecord()));
     }
 }

hedera-state-validator/src/main/java/com/hedera/statevalidation/parameterresolver/VirtualMapAndDataSourceRecord.java

@@ -2,19 +2,9 @@
 package com.hedera.statevalidation.parameterresolver;
 
 import com.swirlds.merkledb.MerkleDbDataSource;
-import com.swirlds.virtualmap.VirtualKey;
 import com.swirlds.virtualmap.VirtualMap;
-import com.swirlds.virtualmap.VirtualValue;
-import com.swirlds.virtualmap.serialize.KeySerializer;
-import com.swirlds.virtualmap.serialize.ValueSerializer;
-
-public record VirtualMapAndDataSourceRecord<K extends VirtualKey, V extends VirtualValue>(
-        String name,
-        MerkleDbDataSource dataSource,
-        VirtualMap map,
-        KeySerializer<K> keySerializer,
-        ValueSerializer<V> valueSerializer) {
 
+public record VirtualMapAndDataSourceRecord(String name, MerkleDbDataSource dataSource, VirtualMap map) {
     @Override
     public String toString() {
         return name;

hedera-state-validator/src/main/java/com/hedera/statevalidation/parameterresolver/VirtualMapHolder.java

@@ -2,35 +2,16 @@
 package com.hedera.statevalidation.parameterresolver;
 
 import static com.hedera.statevalidation.parameterresolver.InitUtils.initConfiguration;
-import static com.hedera.statevalidation.parameterresolver.InitUtils.initServiceRegistry;
-import static com.hedera.statevalidation.parameterresolver.InitUtils.initVirtualMapRecords;
-import static com.hedera.statevalidation.validators.Constants.STATE_DIR;
-
-import com.hedera.node.app.services.ServicesRegistryImpl;
-import com.swirlds.merkledb.MerkleDb;
-import com.swirlds.merkledb.MerkleDbTableConfig;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import static com.hedera.statevalidation.parameterresolver.InitUtils.initVirtualMapRecord;
 
 public class VirtualMapHolder {
     private static VirtualMapHolder instance;
-    private final List<VirtualMapAndDataSourceRecord<?, ?>> records;
-    private final Map<String, MerkleDbTableConfig> tableConfigByNames;
+    private final VirtualMapAndDataSourceRecord record;
 
     private VirtualMapHolder() {
         initConfiguration();
-
-        final ServicesRegistryImpl servicesRegistry = initServiceRegistry();
-
-        final Path stateDirPath = Paths.get(STATE_DIR);
-        final MerkleDb merkleDb = MerkleDb.getInstance(stateDirPath, InitUtils.CONFIGURATION);
-        tableConfigByNames = merkleDb.getTableConfigs();
-
         try {
-            records = initVirtualMapRecords(servicesRegistry);
+            record = initVirtualMapRecord();
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
@@ -41,15 +22,7 @@ public static VirtualMapHolder getInstance() {
         return instance;
     }
 
-    public List<String> getTableNames() {
-        return new ArrayList<>(tableConfigByNames.keySet());
-    }
-
-    public List<VirtualMapAndDataSourceRecord<?, ?>> getRecords() {
-        return records;
-    }
-
-    public Map<String, MerkleDbTableConfig> getTableConfigByNames() {
-        return tableConfigByNames;
+    public VirtualMapAndDataSourceRecord getRecord() {
+        return record;
     }
 }

hedera-state-validator/src/main/java/com/hedera/statevalidation/reporting/Report.java

@@ -55,4 +55,30 @@ public Map<String, VirtualMapReport> getVmapReportByName() {
     public void setVmapReportByName(final Map<String, VirtualMapReport> vmapReportByName) {
         this.vmapReportByName = vmapReportByName;
     }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("REPORT SUMMARY\n");
+        sb.append("=============\n");
+        sb.append(String.format("Node: %s\n", nodeName));
+        sb.append(String.format("Round Number: %d\n", roundNumber));
+        sb.append(String.format("Number of Accounts: %d\n", numberOfAccounts));
+
+        sb.append("\nSTATE REPORT\n");
+        sb.append(stateReport.toString());
+
+        sb.append("\nVIRTUAL MAP REPORTS\n");
+        sb.append("==================\n");
+        if (vmapReportByName.isEmpty()) {
+            sb.append("No virtual map reports available.\n");
+        } else {
+            for (Map.Entry<String, VirtualMapReport> entry : vmapReportByName.entrySet()) {
+                sb.append(String.format("\n[%s]\n", entry.getKey()));
+                sb.append(entry.getValue().toString());
+            }
+        }
+
+        return sb.toString();
+    }
 }

hedera-state-validator/src/main/java/com/hedera/statevalidation/reporting/StateReport.java

@@ -24,4 +24,20 @@ public void setRootHash(final String rootHash) {
     public void setCalculatedHash(final String calculatedHash) {
         this.calculatedHash = calculatedHash;
     }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("============\n");
+
+        sb.append(String.format("Root Hash: %s\n", rootHash));
+        sb.append(String.format("Calculated Hash: %s\n", calculatedHash));
+
+        if (rootHash != null && calculatedHash != null) {
+            boolean hashesMatch = rootHash.equals(calculatedHash);
+            sb.append(String.format("Hashes Match: %s\n", hashesMatch ? "YES" : "NO"));
+        }
+
+        return sb.toString();
+    }
 }

hedera-state-validator/src/main/java/com/hedera/statevalidation/reporting/StorageReport.java

@@ -67,4 +67,22 @@ public long itemCount() {
     public void setItemCount(final long itemCount) {
         this.itemCount = itemCount;
     }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append(String.format("  Path Range: %d to %d\n", minPath, maxPath));
+        sb.append(String.format("  Size: %d MB\n", onDiskSizeInMb));
+        sb.append(String.format("  Files: %d\n", numberOfStorageFiles));
+        sb.append(String.format("  Items: %,d\n", itemCount));
+
+        sb.append(String.format("  Waste: %.2f%%\n", wastePercentage));
+
+        if (duplicateItems > 0) {
+            sb.append(String.format("  Duplicates: %,d\n", duplicateItems));
+        }
+
+        return sb.toString();
+    }
 }