diff --git a/app/app.go b/app/app.go index efdd9f13..3ae49030 100644 --- a/app/app.go +++ b/app/app.go @@ -13,6 +13,7 @@ import ( "github.com/cosmos/cosmos-sdk/std" "github.com/cosmos/cosmos-sdk/x/auth" + authsims "github.com/cosmos/cosmos-sdk/x/auth/simulation" "github.com/cosmos/cosmos-sdk/x/bank" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" "github.com/cosmos/cosmos-sdk/x/consensus" @@ -366,6 +367,13 @@ func NewHeimdallApp( testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{}) + overrideModules := map[string]module.AppModuleSimulation{ + authtypes.ModuleName: auth.NewAppModule(app.appCodec, app.AccountKeeper, authsims.RandomGenesisAccounts, nil), + } + app.simulationManager = module.NewSimulationManagerFromAppModules(app.mm.Modules, overrideModules) + + app.simulationManager.RegisterStoreDecoders() + // initialize stores app.MountKVStores(keys) app.MountTransientStores(tkeys) diff --git a/app/sim_bench_test.go b/app/sim_bench_test.go new file mode 100644 index 00000000..bf9d433d --- /dev/null +++ b/app/sim_bench_test.go @@ -0,0 +1,93 @@ +package app + +import ( + "os" + "testing" + + flag "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" + simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + simtypes "github.com/cosmos/cosmos-sdk/types/simulation" + "github.com/cosmos/cosmos-sdk/x/simulation" + simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" +) + +var FlagEnableBenchStreamingValue bool + +// Get flags every time the simulator is run +func init() { + flag.BoolVar(&FlagEnableBenchStreamingValue, "EnableStreaming", false, "Enable streaming service") +} + +// TODO HV2: test this once https://github.com/0xPolygon/cosmos-sdk/pull/3 is merged +// see https://github.com/0xPolygon/heimdall-v2/pull/10#discussion_r1482779246 + +// Profile with: +// /usr/local/go/bin/go test -benchmem -run=^$ github.com/0xPolygon/heimdall-v2/app -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out +func BenchmarkFullAppSimulation(b *testing.B) { + b.ReportAllocs() + + config := simcli.NewConfigFromFlags() + config.ChainID = HeimdallAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-heimdall", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + if err != nil { + b.Fatalf("simulation setup failed: %s", err.Error()) + } + + if skip { + b.Skip("skipping benchmark application simulation") + } + + defer func() { + require.NoError(b, db.Close()) + require.NoError(b, os.RemoveAll(dir)) + }() + + appOptions := viper.New() + if FlagEnableStreamingValue { + m := make(map[string]interface{}) + m["streaming.abci.keys"] = []string{"*"} + m["streaming.abci.plugin"] = "abci_v1" //nolint:goconst + m["streaming.abci.stop-node-on-err"] = true + for key, value := range m { + appOptions.SetDefault(key, value) + } + } + appOptions.SetDefault(flags.FlagHome, DefaultNodeHome) + appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue) + + app := NewHeimdallApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(HeimdallAppChainID)) + + // run randomized simulation + + _, simParams, simErr := simulation.SimulateFromSeed( + b, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.BlockedModuleAccountAddrs(app.ModuleAccountAddrs()), + config, + app.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil { + b.Fatal(err) + } + + if simErr != nil { + b.Fatal(simErr) + } + + if config.Commit { + simtestutil.PrintStats(db) + } +} diff --git a/app/sim_test.go b/app/sim_test.go new file mode 100644 index 00000000..c896c8f0 --- /dev/null +++ b/app/sim_test.go @@ -0,0 +1,391 @@ +package app + +import ( + "flag" + "fmt" + "math/rand" + "os" + "runtime/debug" + "strings" + "testing" + + abci "github.com/cometbft/cometbft/abci/types" + cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" + dbm "github.com/cosmos/cosmos-db" + jsoniter "github.com/json-iterator/go" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "cosmossdk.io/log" + "cosmossdk.io/store" + storetypes "cosmossdk.io/store/types" + + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" + simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + simtypes "github.com/cosmos/cosmos-sdk/types/simulation" + "github.com/cosmos/cosmos-sdk/x/simulation" + simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" +) + +// HeimdallAppChainID hardcoded chainID for simulation +const HeimdallAppChainID = "simulation-app" + +var FlagEnableStreamingValue bool + +// Get flags every time the simulator is run +func init() { + simcli.GetSimulatorFlags() + flag.BoolVar(&FlagEnableStreamingValue, "EnableStreaming", false, "Enable streaming service") +} + +// fauxMerkleModeOpt returns a BaseApp option to use a dbStoreAdapter instead of +// an IAVLStore for faster simulation speed. +func fauxMerkleModeOpt(bapp *baseapp.BaseApp) { + bapp.SetFauxMerkleMode() +} + +// interBlockCacheOpt returns a BaseApp option function that sets the persistent +// inter-block write-through cache. +func interBlockCacheOpt() func(*baseapp.BaseApp) { + return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager()) +} + +//nolint:paralleltest +func TestFullAppSimulation(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = HeimdallAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-heimdall", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + if skip { + t.Skip("skipping application simulation") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + app := NewHeimdallApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(HeimdallAppChainID)) + require.Equal(t, "HeimdallApp", app.Name()) + + // run randomized simulation + _, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.BlockedModuleAccountAddrs(app.ModuleAccountAddrs()), + config, + app.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } +} + +//nolint:paralleltest +func TestAppImportExport(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = HeimdallAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-heimdall", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + if skip { + t.Skip("skipping application import/export simulation") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + app := NewHeimdallApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(HeimdallAppChainID)) + require.Equal(t, "HeimdallApp", app.Name()) + + // Run randomized simulation + _, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.BlockedModuleAccountAddrs(app.ModuleAccountAddrs()), + config, + app.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } + + fmt.Printf("exporting genesis...\n") + + // TODO CHECK HEIMDALL-V2: We should add the relevant modules when implemented in the modulesToExport list + exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) + require.NoError(t, err) + + fmt.Printf("importing genesis...\n") + + newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-heimdall-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, newDB.Close()) + require.NoError(t, os.RemoveAll(newDir)) + }() + + newApp := NewHeimdallApp(log.NewNopLogger(), newDB, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(HeimdallAppChainID)) + require.Equal(t, "HeimdallApp", newApp.Name()) + + var genesisState GenesisState + err = jsoniter.ConfigFastest.Unmarshal(exported.AppState, &genesisState) + require.NoError(t, err) + + ctxA := app.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()}) + ctxB := newApp.NewContextLegacy(true, cmtproto.Header{Height: app.LastBlockHeight()}) + _, err = newApp.mm.InitGenesis(ctxB, app.AppCodec(), genesisState) + + if err != nil { + if strings.Contains(err.Error(), "validator set is empty after InitGenesis") { + logger.Info("Skipping simulation as all validators have been unbonded") + logger.Info("err", err, "stacktrace", string(debug.Stack())) + return + } + } + + require.NoError(t, err) + err = newApp.StoreConsensusParams(ctxB, exported.ConsensusParams) + require.NoError(t, err) + fmt.Printf("comparing stores...\n") + + storeKeys := app.GetStoreKeys() + require.NotEmpty(t, storeKeys) + + for _, appKeyA := range storeKeys { + // only compare kvstores + if _, ok := appKeyA.(*storetypes.KVStoreKey); !ok { + continue + } + + keyName := appKeyA.Name() + appKeyB := newApp.GetKey(keyName) + + storeA := ctxA.KVStore(appKeyA) + storeB := ctxB.KVStore(appKeyB) + + failedKVAs, failedKVBs := simtestutil.DiffKVStores(storeA, storeB, nil) + require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare %s", keyName) + + fmt.Printf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), appKeyA, appKeyB) + + require.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(keyName, app.SimulationManager().StoreDecoders, failedKVAs, failedKVBs)) + } +} + +//nolint:paralleltest +func TestAppSimulationAfterImport(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = HeimdallAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "leveldb-app-heimdall", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + if skip { + t.Skip("skipping application simulation after import") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + app := NewHeimdallApp(logger, db, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(HeimdallAppChainID)) + require.Equal(t, "HeimdallApp", app.Name()) + + // Run randomized simulation + stopEarly, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.BlockedModuleAccountAddrs(app.ModuleAccountAddrs()), + config, + app.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } + + if stopEarly { + fmt.Println("can't export or import a zero-validator genesis, exiting test...") + return + } + + fmt.Printf("exporting genesis...\n") + + // TODO CHECK HEIMDALL-V2: We should add the relevant modules when implemented in the modulesToExport list + exported, err := app.ExportAppStateAndValidators(false, []string{}, []string{}) + require.NoError(t, err) + + fmt.Printf("importing genesis...\n") + + newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-heimdall-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, newDB.Close()) + require.NoError(t, os.RemoveAll(newDir)) + }() + + newApp := NewHeimdallApp(log.NewNopLogger(), newDB, nil, true, appOptions, fauxMerkleModeOpt, baseapp.SetChainID(HeimdallAppChainID)) + require.Equal(t, "HeimdallApp", newApp.Name()) + + _, err = newApp.InitChain(&abci.RequestInitChain{ + AppStateBytes: exported.AppState, + ChainId: HeimdallAppChainID, + }) + require.NoError(t, err) + + _, _, err = simulation.SimulateFromSeed( + t, + os.Stdout, + newApp.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(newApp, newApp.AppCodec(), config), + app.BlockedModuleAccountAddrs(app.ModuleAccountAddrs()), + config, + app.AppCodec(), + ) + require.NoError(t, err) +} + +// TODO: Make another test for the fuzzer itself, which just has noOp txs +// and doesn't depend on the application. +// +//nolint:paralleltest +func TestAppStateDeterminism(t *testing.T) { + if !simcli.FlagEnabledValue { + t.Skip("skipping application simulation") + } + + config := simcli.NewConfigFromFlags() + config.InitialBlockHeight = 1 + config.ExportParamsPath = "" + config.OnOperation = false + config.AllInvariants = false + config.ChainID = HeimdallAppChainID + + numSeeds := 3 + numTimesToRunPerSeed := 3 + appHashList := make([]jsoniter.RawMessage, numTimesToRunPerSeed) + + // We will be overriding the random seed and just run a single simulation on the provided seed value + if config.Seed != simcli.DefaultSeedValue { + numSeeds = 1 + } + + appOptions := viper.New() + if FlagEnableStreamingValue { + m := make(map[string]interface{}) + m["streaming.abci.keys"] = []string{"*"} + m["streaming.abci.plugin"] = "abci_v1" + m["streaming.abci.stop-node-on-err"] = true + for key, value := range m { + appOptions.SetDefault(key, value) + } + } + appOptions.SetDefault(flags.FlagHome, DefaultNodeHome) + appOptions.SetDefault(server.FlagInvCheckPeriod, simcli.FlagPeriodValue) + if simcli.FlagVerboseValue { + appOptions.SetDefault(flags.FlagLogLevel, "debug") + } + + for i := 0; i < numSeeds; i++ { + if config.Seed == simcli.DefaultSeedValue { + config.Seed = rand.Int63() //nolint:gosec + } + + fmt.Println("config.Seed: ", config.Seed) + + for j := 0; j < numTimesToRunPerSeed; j++ { + var logger log.Logger + if simcli.FlagVerboseValue { + logger = log.NewTestLogger(t) + } else { + logger = log.NewNopLogger() + } + + db := dbm.NewMemDB() + app := NewHeimdallApp(logger, db, nil, true, appOptions, interBlockCacheOpt(), baseapp.SetChainID(HeimdallAppChainID)) + + fmt.Printf( + "running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n", + config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, + ) + + _, _, err := simulation.SimulateFromSeed( + t, + os.Stdout, + app.BaseApp, + simtestutil.AppStateFn(app.AppCodec(), app.SimulationManager(), app.DefaultGenesis()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(app, app.AppCodec(), config), + app.BlockedModuleAccountAddrs(app.ModuleAccountAddrs()), + config, + app.AppCodec(), + ) + require.NoError(t, err) + + if config.Commit { + simtestutil.PrintStats(db) + } + + appHash := app.LastCommitID().Hash + appHashList[j] = appHash + + if j != 0 { + require.Equal( + t, string(appHashList[0]), string(appHashList[j]), + "non-determinism in seed %d: %d/%d, attempt: %d/%d\n", config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, + ) + } + } + } +} diff --git a/go.mod b/go.mod index 79c796dd..0e90ce18 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,10 @@ require ( github.com/cosmos/cosmos-sdk v0.50.1 github.com/cosmos/gogoproto v1.4.11 github.com/ethereum/go-ethereum v1.13.14 + github.com/json-iterator/go v1.1.12 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.18.1 + github.com/stretchr/testify v1.8.4 ) require ( @@ -148,6 +152,8 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect @@ -179,10 +185,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.18.1 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect - github.com/stretchr/testify v1.8.4 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect