From ba9112aaeb36cd43141822d5075aa9f82203316e Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Wed, 16 Oct 2024 19:15:12 -0300 Subject: [PATCH 001/135] Add first version of retry connection in Go --- batcher/Cargo.lock | 2 +- core/connection.go | 41 ++++++++++++++++++++++++++++++++++++++++ core/connection_test.go | 28 +++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 ++ operator/sp1/sp1_test.go | 1 - 6 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 core/connection.go create mode 100644 core/connection_test.go diff --git a/batcher/Cargo.lock b/batcher/Cargo.lock index 210d70bdc..bf873c309 100644 --- a/batcher/Cargo.lock +++ b/batcher/Cargo.lock @@ -78,7 +78,7 @@ dependencies = [ [[package]] name = "aligned" -version = "0.8.0" +version = "0.9.2" dependencies = [ "aligned-sdk", "clap", diff --git a/core/connection.go b/core/connection.go new file mode 100644 index 000000000..07afadfa1 --- /dev/null +++ b/core/connection.go @@ -0,0 +1,41 @@ +package connection + +import ( + "fmt" + "time" + + "github.com/cenkalti/backoff/v4" +) + +type PermanentError struct { + Inner error +} + +func (e PermanentError) Error() string { return e.Inner.Error() } +func (e PermanentError) Unwrap() error { + return e.Inner +} +func (e PermanentError) Is(err error) bool { + _, ok := err.(PermanentError) + return ok +} + +func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor float64, maxTries uint64) (interface{}, error) { + f := func() (interface{}, error) { + val, err := functionToRetry() + if perm, ok := err.(PermanentError); err != nil && ok { + fmt.Println("ENTRE") + return nil, backoff.Permanent(perm.Inner) + } + return val, err + } + + randomOption := backoff.WithRandomizationFactor(0) + + initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) + multiplierOption := backoff.WithMultiplier(factor) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) + maxRetriesBackoff := backoff.WithMaxRetries(expBackoff, maxTries) + + return backoff.RetryWithData(f, maxRetriesBackoff) +} diff --git a/core/connection_test.go b/core/connection_test.go new file mode 100644 index 000000000..73f6b9893 --- /dev/null +++ b/core/connection_test.go @@ -0,0 +1,28 @@ +package connection_test + +import ( + "fmt" + "testing" + + "github.com/yetanotherco/aligned_layer/core" +) + +func DummyFunction(x uint64) (uint64, error) { + fmt.Println("Doing some work...") + if x == 42 { + return 0, connection.PermanentError{Inner: fmt.Errorf("Permanent error!")} + } else if x < 42 { + return 0, fmt.Errorf("Transient error!") + } + return x, nil +} + +func TestRetry(t *testing.T) { + function := func() (interface{}, error) { return DummyFunction(42) } + data, err := connection.Retry(function, 1000, 2, 3) + if err != nil { + t.Errorf("Retry error!: %s", err) + } else { + fmt.Printf("DATA: %d\n", data) + } +} diff --git a/go.mod b/go.mod index 502e791b0..c32b26693 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect diff --git a/go.sum b/go.sum index dceaadc81..f5d8920a4 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/operator/sp1/sp1_test.go b/operator/sp1/sp1_test.go index d342751bd..3b9224729 100644 --- a/operator/sp1/sp1_test.go +++ b/operator/sp1/sp1_test.go @@ -8,7 +8,6 @@ import ( ) const ProofFilePath = "../../scripts/test_files/sp1/sp1_fibonacci.proof" - const ElfFilePath = "../../scripts/test_files/sp1/sp1_fibonacci.elf" func TestFibonacciSp1ProofVerifies(t *testing.T) { From 42b4ed3d0cf1d78966e6d7898ea678c3fc6528d7 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Wed, 16 Oct 2024 19:20:57 -0300 Subject: [PATCH 002/135] Add some docs --- core/connection.go | 7 +++++-- core/connection_test.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/connection.go b/core/connection.go index 07afadfa1..0a1494ff1 100644 --- a/core/connection.go +++ b/core/connection.go @@ -1,7 +1,6 @@ package connection import ( - "fmt" "time" "github.com/cenkalti/backoff/v4" @@ -20,11 +19,15 @@ func (e PermanentError) Is(err error) bool { return ok } +// Retries a given function in an exponential backoff manner. +// It will retry calling the function while it returns an error, until the max retries +// from the configuration are reached, or until a `PermanentError` is returned. +// The function to be retried should return `PermanentError` when the condition for stop retrying +// is met. func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor float64, maxTries uint64) (interface{}, error) { f := func() (interface{}, error) { val, err := functionToRetry() if perm, ok := err.(PermanentError); err != nil && ok { - fmt.Println("ENTRE") return nil, backoff.Permanent(perm.Inner) } return val, err diff --git a/core/connection_test.go b/core/connection_test.go index 73f6b9893..b822ddcc3 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -18,7 +18,7 @@ func DummyFunction(x uint64) (uint64, error) { } func TestRetry(t *testing.T) { - function := func() (interface{}, error) { return DummyFunction(42) } + function := func() (interface{}, error) { return DummyFunction(43) } data, err := connection.Retry(function, 1000, 2, 3) if err != nil { t.Errorf("Retry error!: %s", err) From 896b98c9af6963d1b3614903837f275464a24a9d Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 17 Oct 2024 15:23:07 -0300 Subject: [PATCH 003/135] feat: basic aggregator bump modelling --- aggregator/internal/pkg/aggregator.go | 9 +---- core/chainio/avs_writer.go | 56 ++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index f17591edc..6568d2106 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -301,7 +301,7 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "senderAddress", hex.EncodeToString(senderAddress[:]), "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) - txHash, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Error sending aggregated response for batch %s. Error: %s", hex.EncodeToString(batchIdentifierHash[:]), err) @@ -312,13 +312,6 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Sending aggregated response for batch %s", hex.EncodeToString(batchIdentifierHash[:])) - receipt, err := utils.WaitForTransactionReceipt( - agg.AggregatorConfig.BaseConfig.EthRpcClient, context.Background(), *txHash) - if err != nil { - agg.telemetry.LogTaskError(batchMerkleRoot, err) - return nil, err - } - agg.metrics.IncAggregatedResponses() return receipt, nil diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index a6a820195..51a4d6d58 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -11,11 +11,13 @@ import ( "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/Layr-Labs/eigensdk-go/signer" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" "github.com/yetanotherco/aligned_layer/core/config" + "github.com/yetanotherco/aligned_layer/core/utils" ) type AvsWriter struct { @@ -70,7 +72,7 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E }, nil } -func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*common.Hash, error) { +func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) @@ -87,21 +89,55 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, err } - // Send the transaction + // Set the nonce, as we might have to replace the transaction with a higher fee + txNonce := new(big.Int).SetUint64(tx.Nonce()) txOpts.NoSend = false - txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit - tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - // Retry with fallback - tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + txOpts.Nonce = txNonce + + // Send the transaction + var maxRetries uint64 = 5 + var i uint64 + for i = 1; i < maxRetries; i++ { + // Add x% to the gas limit, where x 10 <= x <= 50 + txOpts.GasLimit = tx.Gas() * (100 + i*10) / 100 + w.logger.Debugf("Sending ResponseToTask transaction for %vth with a gas limit of %v", i, txOpts.GasLimit) + err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { return nil, err } - } - txHash := tx.Hash() + tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + // Retry with fallback + tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + return nil, err + } + } + ctx, _ := context.WithTimeout(context.Background(), time.Second*20) + receipt, err := utils.WaitForTransactionReceipt(w.Client, ctx, tx.Hash()) + + if receipt != nil { + if receipt.Status == 0 { + return receipt, fmt.Errorf("transaction failed") + } else { + // transaction was included in block + w.logger.Debugf("Sending ResponseToTask transaction for %vth with a gas limit of %v", i, txOpts.GasLimit) + return receipt, nil + } + } + + // transaction not included in block, try again + if err == ethereum.NotFound { + w.logger.Debugf("Transaction not included in block will try again") + continue + } else { + return receipt, err + } + + } - return &txHash, nil + return nil, fmt.Errorf("could not send transaction") } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { From dceaa0a381fa97fa7d0f36e29c1d11ca13550c31 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 17 Oct 2024 17:52:11 -0300 Subject: [PATCH 004/135] fix: gas limit for gas fee cap --- core/chainio/avs_writer.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 51a4d6d58..6c607df6a 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -76,6 +76,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { // Retry with fallback tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) @@ -98,9 +99,10 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe var maxRetries uint64 = 5 var i uint64 for i = 1; i < maxRetries; i++ { - // Add x% to the gas limit, where x 10 <= x <= 50 - txOpts.GasLimit = tx.Gas() * (100 + i*10) / 100 - w.logger.Debugf("Sending ResponseToTask transaction for %vth with a gas limit of %v", i, txOpts.GasLimit) + // factor = (100 + i * 10) / 100, so 1,1 <= x <= 1,5 + factor := new(big.Int).Div(new(big.Int).Add(big.NewInt(100), new(big.Int).Mul(big.NewInt(int64(i)), big.NewInt(10))), big.NewInt(100)) + txOpts.GasFeeCap = new(big.Int).Mul(new(big.Int).Sub(tx.GasFeeCap(), big.NewInt(int64(1000))), factor) + w.logger.Infof("Sending ResponseToTask transaction for %vth with a gas limit of %v", i, txOpts.GasLimit) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { return nil, err @@ -122,14 +124,13 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return receipt, fmt.Errorf("transaction failed") } else { // transaction was included in block - w.logger.Debugf("Sending ResponseToTask transaction for %vth with a gas limit of %v", i, txOpts.GasLimit) return receipt, nil } } // transaction not included in block, try again if err == ethereum.NotFound { - w.logger.Debugf("Transaction not included in block will try again") + w.logger.Infof("Transaction not included in block will try again bumping the fee") continue } else { return receipt, err From 61a64df948f5c96d0e2fe7cadcade0df2c4d0a1d Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Fri, 18 Oct 2024 10:38:42 -0300 Subject: [PATCH 005/135] fix: gas limit calculation --- core/chainio/avs_writer.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 6c607df6a..933c1085f 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -99,10 +99,14 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe var maxRetries uint64 = 5 var i uint64 for i = 1; i < maxRetries; i++ { + // bump the fee here // factor = (100 + i * 10) / 100, so 1,1 <= x <= 1,5 - factor := new(big.Int).Div(new(big.Int).Add(big.NewInt(100), new(big.Int).Mul(big.NewInt(int64(i)), big.NewInt(10))), big.NewInt(100)) - txOpts.GasFeeCap = new(big.Int).Mul(new(big.Int).Sub(tx.GasFeeCap(), big.NewInt(int64(1000))), factor) - w.logger.Infof("Sending ResponseToTask transaction for %vth with a gas limit of %v", i, txOpts.GasLimit) + factor := (new(big.Int).Add(big.NewInt(100), new(big.Int).Mul(big.NewInt(int64(i)), big.NewInt(10)))) + gasPrice := new(big.Int).Mul(tx.GasPrice(), factor) + gasPrice = gasPrice.Div(gasPrice, big.NewInt(100)) + txOpts.GasPrice = gasPrice + + w.logger.Infof("Sending ResponseToTask transaction for %vth with a gas price of %v", i, txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { return nil, err @@ -116,7 +120,8 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, err } } - ctx, _ := context.WithTimeout(context.Background(), time.Second*20) + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*10) + defer cancel() receipt, err := utils.WaitForTransactionReceipt(w.Client, ctx, tx.Hash()) if receipt != nil { @@ -132,6 +137,8 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe if err == ethereum.NotFound { w.logger.Infof("Transaction not included in block will try again bumping the fee") continue + } else if err == context.DeadlineExceeded { + continue } else { return receipt, err } From 6873cbeb314069b09370e99d092d69972b89a598 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Fri, 18 Oct 2024 15:25:49 -0300 Subject: [PATCH 006/135] fix: exiting tx receipt wait when context times out --- core/chainio/avs_writer.go | 13 ++++--------- core/utils/eth_client_utils.go | 4 ++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 933c1085f..d30a5e71f 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -11,7 +11,6 @@ import ( "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/Layr-Labs/eigensdk-go/signer" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -120,7 +119,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, err } } - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*36) defer cancel() receipt, err := utils.WaitForTransactionReceipt(w.Client, ctx, tx.Hash()) @@ -133,14 +132,10 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe } } - // transaction not included in block, try again - if err == ethereum.NotFound { - w.logger.Infof("Transaction not included in block will try again bumping the fee") - continue - } else if err == context.DeadlineExceeded { + // this means we have reached the timeout (after three blocks it hasn't been included) + // so we try again by bumping the fee to make sure its included + if err != nil { continue - } else { - return receipt, err } } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index afa26b867..792f34718 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -18,6 +18,10 @@ const sleepTime = 5 * time.Second func WaitForTransactionReceipt(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { for i := 0; i < maxRetries; i++ { receipt, err := client.TransactionReceipt(ctx, txHash) + // if context has timed out, return + if ctx.Err() != nil { + return nil, ctx.Err() + } if err != nil { time.Sleep(sleepTime) } else { From b5bf04e893a5f8e48ba1b830037f6381268cc10a Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 15:45:26 -0300 Subject: [PATCH 007/135] feat: retryWithData and without, generic pointer for return type, infinite retries --- core/connection.go | 45 ++++++++++++++++++++++++++++++++++------- core/connection_test.go | 22 ++++++++++++++++---- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/core/connection.go b/core/connection.go index 0a1494ff1..b1a9f4f14 100644 --- a/core/connection.go +++ b/core/connection.go @@ -19,18 +19,49 @@ func (e PermanentError) Is(err error) bool { return ok } +// Same as Retry only that the functionToRetry can return a value upon correct execution +func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { + i := 0 + f := func() (*T, error) { + val, err := functionToRetry() + i++ + if perm, ok := err.(PermanentError); err != nil && ok { + return nil, backoff.Permanent(perm.Inner) + } + return val, err + } + + randomOption := backoff.WithRandomizationFactor(0) + + initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) + multiplierOption := backoff.WithMultiplier(factor) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) + var maxRetriesBackoff backoff.BackOff + + if maxTries > 0 { + maxRetriesBackoff = backoff.WithMaxRetries(expBackoff, maxTries) + } else { + maxRetriesBackoff = expBackoff + } + + return backoff.RetryWithData(f, maxRetriesBackoff) +} + // Retries a given function in an exponential backoff manner. -// It will retry calling the function while it returns an error, until the max retries +// It will retry calling the function while it returns an error, until the max retries. +// If maxTries == 0 then the retry function will run indefinitely until success // from the configuration are reached, or until a `PermanentError` is returned. // The function to be retried should return `PermanentError` when the condition for stop retrying // is met. -func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor float64, maxTries uint64) (interface{}, error) { - f := func() (interface{}, error) { - val, err := functionToRetry() +func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64) error { + i := 0 + f := func() error { + err := functionToRetry() + i++ if perm, ok := err.(PermanentError); err != nil && ok { - return nil, backoff.Permanent(perm.Inner) + return backoff.Permanent(perm.Inner) } - return val, err + return err } randomOption := backoff.WithRandomizationFactor(0) @@ -40,5 +71,5 @@ func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) maxRetriesBackoff := backoff.WithMaxRetries(expBackoff, maxTries) - return backoff.RetryWithData(f, maxRetriesBackoff) + return backoff.Retry(f, maxRetriesBackoff) } diff --git a/core/connection_test.go b/core/connection_test.go index b822ddcc3..a922ec10e 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/yetanotherco/aligned_layer/core" + connection "github.com/yetanotherco/aligned_layer/core" ) func DummyFunction(x uint64) (uint64, error) { @@ -17,12 +17,26 @@ func DummyFunction(x uint64) (uint64, error) { return x, nil } -func TestRetry(t *testing.T) { - function := func() (interface{}, error) { return DummyFunction(43) } - data, err := connection.Retry(function, 1000, 2, 3) +func TestRetryWithData(t *testing.T) { + function := func() (*uint64, error) { + x, err := DummyFunction(43) + return &x, err + } + data, err := connection.RetryWithData(function, 1000, 2, 3) if err != nil { t.Errorf("Retry error!: %s", err) } else { fmt.Printf("DATA: %d\n", data) } } + +func TestRetry(t *testing.T) { + function := func() error { + _, err := DummyFunction(43) + return err + } + err := connection.Retry(function, 1000, 2, 3) + if err != nil { + t.Errorf("Retry error!: %s", err) + } +} From 8bfd336a1d68472fdaee1aca190e033f35cd2147 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Wed, 16 Oct 2024 19:15:12 -0300 Subject: [PATCH 008/135] Add first version of retry connection in Go --- core/connection.go | 41 ++++++++++++++++++++++++++++++++++++++++ core/connection_test.go | 28 +++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 ++ operator/sp1/sp1_test.go | 1 - 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 core/connection.go create mode 100644 core/connection_test.go diff --git a/core/connection.go b/core/connection.go new file mode 100644 index 000000000..07afadfa1 --- /dev/null +++ b/core/connection.go @@ -0,0 +1,41 @@ +package connection + +import ( + "fmt" + "time" + + "github.com/cenkalti/backoff/v4" +) + +type PermanentError struct { + Inner error +} + +func (e PermanentError) Error() string { return e.Inner.Error() } +func (e PermanentError) Unwrap() error { + return e.Inner +} +func (e PermanentError) Is(err error) bool { + _, ok := err.(PermanentError) + return ok +} + +func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor float64, maxTries uint64) (interface{}, error) { + f := func() (interface{}, error) { + val, err := functionToRetry() + if perm, ok := err.(PermanentError); err != nil && ok { + fmt.Println("ENTRE") + return nil, backoff.Permanent(perm.Inner) + } + return val, err + } + + randomOption := backoff.WithRandomizationFactor(0) + + initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) + multiplierOption := backoff.WithMultiplier(factor) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) + maxRetriesBackoff := backoff.WithMaxRetries(expBackoff, maxTries) + + return backoff.RetryWithData(f, maxRetriesBackoff) +} diff --git a/core/connection_test.go b/core/connection_test.go new file mode 100644 index 000000000..73f6b9893 --- /dev/null +++ b/core/connection_test.go @@ -0,0 +1,28 @@ +package connection_test + +import ( + "fmt" + "testing" + + "github.com/yetanotherco/aligned_layer/core" +) + +func DummyFunction(x uint64) (uint64, error) { + fmt.Println("Doing some work...") + if x == 42 { + return 0, connection.PermanentError{Inner: fmt.Errorf("Permanent error!")} + } else if x < 42 { + return 0, fmt.Errorf("Transient error!") + } + return x, nil +} + +func TestRetry(t *testing.T) { + function := func() (interface{}, error) { return DummyFunction(42) } + data, err := connection.Retry(function, 1000, 2, 3) + if err != nil { + t.Errorf("Retry error!: %s", err) + } else { + fmt.Printf("DATA: %d\n", data) + } +} diff --git a/go.mod b/go.mod index 502e791b0..c32b26693 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect diff --git a/go.sum b/go.sum index dceaadc81..f5d8920a4 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/operator/sp1/sp1_test.go b/operator/sp1/sp1_test.go index d342751bd..3b9224729 100644 --- a/operator/sp1/sp1_test.go +++ b/operator/sp1/sp1_test.go @@ -8,7 +8,6 @@ import ( ) const ProofFilePath = "../../scripts/test_files/sp1/sp1_fibonacci.proof" - const ElfFilePath = "../../scripts/test_files/sp1/sp1_fibonacci.elf" func TestFibonacciSp1ProofVerifies(t *testing.T) { From 9cb678085f47ae941d3fba4ababeeaf7869097cc Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Wed, 16 Oct 2024 19:20:57 -0300 Subject: [PATCH 009/135] Add some docs --- core/connection.go | 7 +++++-- core/connection_test.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/connection.go b/core/connection.go index 07afadfa1..0a1494ff1 100644 --- a/core/connection.go +++ b/core/connection.go @@ -1,7 +1,6 @@ package connection import ( - "fmt" "time" "github.com/cenkalti/backoff/v4" @@ -20,11 +19,15 @@ func (e PermanentError) Is(err error) bool { return ok } +// Retries a given function in an exponential backoff manner. +// It will retry calling the function while it returns an error, until the max retries +// from the configuration are reached, or until a `PermanentError` is returned. +// The function to be retried should return `PermanentError` when the condition for stop retrying +// is met. func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor float64, maxTries uint64) (interface{}, error) { f := func() (interface{}, error) { val, err := functionToRetry() if perm, ok := err.(PermanentError); err != nil && ok { - fmt.Println("ENTRE") return nil, backoff.Permanent(perm.Inner) } return val, err diff --git a/core/connection_test.go b/core/connection_test.go index 73f6b9893..b822ddcc3 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -18,7 +18,7 @@ func DummyFunction(x uint64) (uint64, error) { } func TestRetry(t *testing.T) { - function := func() (interface{}, error) { return DummyFunction(42) } + function := func() (interface{}, error) { return DummyFunction(43) } data, err := connection.Retry(function, 1000, 2, 3) if err != nil { t.Errorf("Retry error!: %s", err) From 262765912e05eb3692f5de70c23477ee157da97c Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 15:45:26 -0300 Subject: [PATCH 010/135] feat: retryWithData and without, generic pointer for return type, infinite retries --- core/connection.go | 45 ++++++++++++++++++++++++++++++++++------- core/connection_test.go | 22 ++++++++++++++++---- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/core/connection.go b/core/connection.go index 0a1494ff1..b1a9f4f14 100644 --- a/core/connection.go +++ b/core/connection.go @@ -19,18 +19,49 @@ func (e PermanentError) Is(err error) bool { return ok } +// Same as Retry only that the functionToRetry can return a value upon correct execution +func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { + i := 0 + f := func() (*T, error) { + val, err := functionToRetry() + i++ + if perm, ok := err.(PermanentError); err != nil && ok { + return nil, backoff.Permanent(perm.Inner) + } + return val, err + } + + randomOption := backoff.WithRandomizationFactor(0) + + initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) + multiplierOption := backoff.WithMultiplier(factor) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) + var maxRetriesBackoff backoff.BackOff + + if maxTries > 0 { + maxRetriesBackoff = backoff.WithMaxRetries(expBackoff, maxTries) + } else { + maxRetriesBackoff = expBackoff + } + + return backoff.RetryWithData(f, maxRetriesBackoff) +} + // Retries a given function in an exponential backoff manner. -// It will retry calling the function while it returns an error, until the max retries +// It will retry calling the function while it returns an error, until the max retries. +// If maxTries == 0 then the retry function will run indefinitely until success // from the configuration are reached, or until a `PermanentError` is returned. // The function to be retried should return `PermanentError` when the condition for stop retrying // is met. -func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor float64, maxTries uint64) (interface{}, error) { - f := func() (interface{}, error) { - val, err := functionToRetry() +func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64) error { + i := 0 + f := func() error { + err := functionToRetry() + i++ if perm, ok := err.(PermanentError); err != nil && ok { - return nil, backoff.Permanent(perm.Inner) + return backoff.Permanent(perm.Inner) } - return val, err + return err } randomOption := backoff.WithRandomizationFactor(0) @@ -40,5 +71,5 @@ func Retry(functionToRetry func() (interface{}, error), minDelay uint64, factor expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) maxRetriesBackoff := backoff.WithMaxRetries(expBackoff, maxTries) - return backoff.RetryWithData(f, maxRetriesBackoff) + return backoff.Retry(f, maxRetriesBackoff) } diff --git a/core/connection_test.go b/core/connection_test.go index b822ddcc3..a922ec10e 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/yetanotherco/aligned_layer/core" + connection "github.com/yetanotherco/aligned_layer/core" ) func DummyFunction(x uint64) (uint64, error) { @@ -17,12 +17,26 @@ func DummyFunction(x uint64) (uint64, error) { return x, nil } -func TestRetry(t *testing.T) { - function := func() (interface{}, error) { return DummyFunction(43) } - data, err := connection.Retry(function, 1000, 2, 3) +func TestRetryWithData(t *testing.T) { + function := func() (*uint64, error) { + x, err := DummyFunction(43) + return &x, err + } + data, err := connection.RetryWithData(function, 1000, 2, 3) if err != nil { t.Errorf("Retry error!: %s", err) } else { fmt.Printf("DATA: %d\n", data) } } + +func TestRetry(t *testing.T) { + function := func() error { + _, err := DummyFunction(43) + return err + } + err := connection.Retry(function, 1000, 2, 3) + if err != nil { + t.Errorf("Retry error!: %s", err) + } +} From 90c4290aecd5228906c2c2e75610388768adde7b Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 16:01:12 -0300 Subject: [PATCH 011/135] feat: SendAggregatedResponse with infinite retry --- core/chainio/avs_writer.go | 36 +++++++++++++++------------------- core/utils/eth_client_utils.go | 11 +++++++++++ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index d30a5e71f..414593d7a 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" + connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/config" "github.com/yetanotherco/aligned_layer/core/utils" ) @@ -93,19 +94,13 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txNonce := new(big.Int).SetUint64(tx.Nonce()) txOpts.NoSend = false txOpts.Nonce = txNonce - - // Send the transaction - var maxRetries uint64 = 5 - var i uint64 - for i = 1; i < maxRetries; i++ { + i := 0 + sendTransaction := func() (*types.Receipt, error) { // bump the fee here - // factor = (100 + i * 10) / 100, so 1,1 <= x <= 1,5 - factor := (new(big.Int).Add(big.NewInt(100), new(big.Int).Mul(big.NewInt(int64(i)), big.NewInt(10)))) - gasPrice := new(big.Int).Mul(tx.GasPrice(), factor) - gasPrice = gasPrice.Div(gasPrice, big.NewInt(100)) + gasPrice := utils.CalculateGasPriceBumpBasedOnRetry(tx.GasPrice(), i) txOpts.GasPrice = gasPrice - w.logger.Infof("Sending ResponseToTask transaction for %vth with a gas price of %v", i, txOpts.GasPrice) + i++ err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { return nil, err @@ -124,23 +119,24 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe receipt, err := utils.WaitForTransactionReceipt(w.Client, ctx, tx.Hash()) if receipt != nil { - if receipt.Status == 0 { - return receipt, fmt.Errorf("transaction failed") - } else { - // transaction was included in block - return receipt, nil - } + return receipt, nil } - // this means we have reached the timeout (after three blocks it hasn't been included) + // if we are here, this means we have reached the timeout (after three blocks it hasn't been included) // so we try again by bumping the fee to make sure its included if err != nil { - continue + return nil, err } - + return nil, fmt.Errorf("transaction failed") } + receipt, err := connection.RetryWithData(sendTransaction, 1000, 2, 3) - return nil, fmt.Errorf("could not send transaction") + if receipt.Status == 0 { + return receipt, fmt.Errorf("transaction failed") + } else { + // transaction was included in block + return receipt, nil + } } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 792f34718..01ddc2a52 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -2,6 +2,7 @@ package utils import ( "context" + "math/big" "time" "fmt" @@ -46,3 +47,13 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e } return quorumThresholdPercentages } + +// Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice and retry iteration. +// It adds a i/10 percentage to the current prices, where i represents the iteration. +func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) *big.Int { + factor := (new(big.Int).Add(big.NewInt(100), new(big.Int).Mul(big.NewInt(int64(iteration)), big.NewInt(10)))) + gasPrice := new(big.Int).Mul(currentGasPrice, factor) + gasPrice = gasPrice.Div(gasPrice, big.NewInt(100)) + + return gasPrice +} From 10597b018cfb9017507ea1b94e00e5f578809e4e Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 16:16:39 -0300 Subject: [PATCH 012/135] feat: retry function with bumping fee --- core/chainio/avs_writer.go | 43 +++++++--------------------------- core/utils/eth_client_utils.go | 38 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 414593d7a..7c85ea3af 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -15,7 +15,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" - connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/config" "github.com/yetanotherco/aligned_layer/core/utils" ) @@ -94,49 +93,25 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txNonce := new(big.Int).SetUint64(tx.Nonce()) txOpts.NoSend = false txOpts.Nonce = txNonce - i := 0 - sendTransaction := func() (*types.Receipt, error) { - // bump the fee here - gasPrice := utils.CalculateGasPriceBumpBasedOnRetry(tx.GasPrice(), i) + + beforeTransaction := func(gasPrice *big.Int) error { txOpts.GasPrice = gasPrice - w.logger.Infof("Sending ResponseToTask transaction for %vth with a gas price of %v", i, txOpts.GasPrice) - i++ + w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) - if err != nil { - return nil, err - } + return err + } + executeTransaction := func(gasPrice *big.Int) (*types.Transaction, error) { tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { // Retry with fallback tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - return nil, err - } + return tx, err } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*36) - defer cancel() - receipt, err := utils.WaitForTransactionReceipt(w.Client, ctx, tx.Hash()) - - if receipt != nil { - return receipt, nil - } - - // if we are here, this means we have reached the timeout (after three blocks it hasn't been included) - // so we try again by bumping the fee to make sure its included - if err != nil { - return nil, err - } - return nil, fmt.Errorf("transaction failed") + return tx, err } - receipt, err := connection.RetryWithData(sendTransaction, 1000, 2, 3) - if receipt.Status == 0 { - return receipt, fmt.Errorf("transaction failed") - } else { - // transaction was included in block - return receipt, nil - } + return utils.SendTransactionWithInfiniteRetryAndBumpingGasPrice(beforeTransaction, executeTransaction, w.Client, tx.GasPrice()) } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 01ddc2a52..af251b3d1 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -11,6 +11,7 @@ import ( eigentypes "github.com/Layr-Labs/eigensdk-go/types" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + connection "github.com/yetanotherco/aligned_layer/core" ) const maxRetries = 25 @@ -57,3 +58,40 @@ func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) return gasPrice } + +// Sends a transaction and waits for the receipt for three blocks, if not received +// it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry`. +// This process happens indefinitely until we get the receipt or the receipt status is an err +func SendTransactionWithInfiniteRetryAndBumpingGasPrice(beforeTransaction func(*big.Int) error, executeTransaction func(*big.Int) (*types.Transaction, error), client eth.InstrumentedClient, baseGasPrice *big.Int) (*types.Receipt, error) { + i := 0 + sendTransaction := func() (*types.Receipt, error) { + i++ + gasPrice := CalculateGasPriceBumpBasedOnRetry(baseGasPrice, i) + + err := beforeTransaction(gasPrice) + if err != nil { + return nil, err + } + + tx, err := executeTransaction(gasPrice) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*36) + defer cancel() + receipt, err := WaitForTransactionReceipt(client, ctx, tx.Hash()) + + if receipt != nil { + return receipt, nil + } + // if we are here, this means we have reached the timeout (after three blocks it hasn't been included) + // so we try again by bumping the fee to make sure its included + if err != nil { + return nil, err + } + return nil, fmt.Errorf("transaction failed") + + } + return connection.RetryWithData(sendTransaction, 1000, 2, 0) +} From dbc7d6021707f75d16ec654d144b8e88552c85c4 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 16:27:49 -0300 Subject: [PATCH 013/135] docs: SendTransactionWithInfiniteRetryAndBumpingGasPrice --- core/utils/eth_client_utils.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index af251b3d1..30755ce23 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -60,8 +60,9 @@ func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) } // Sends a transaction and waits for the receipt for three blocks, if not received -// it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry`. -// This process happens indefinitely until we get the receipt or the receipt status is an err +// it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry` +// and pass it to beforeTransaction and executeTransaction (make sure you update the txOpts with the new price) +// This process happens indefinitely until we get the receipt or the receipt status is an err. func SendTransactionWithInfiniteRetryAndBumpingGasPrice(beforeTransaction func(*big.Int) error, executeTransaction func(*big.Int) (*types.Transaction, error), client eth.InstrumentedClient, baseGasPrice *big.Int) (*types.Receipt, error) { i := 0 sendTransaction := func() (*types.Receipt, error) { From cc9b76519f59fc26fe2f05b732e94b613a65709d Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 17:51:19 -0300 Subject: [PATCH 014/135] refactor: remove beforeTransaction --- core/chainio/avs_writer.go | 13 +++++++------ core/utils/eth_client_utils.go | 7 +------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 7c85ea3af..b538d1d50 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -94,14 +94,15 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts.NoSend = false txOpts.Nonce = txNonce - beforeTransaction := func(gasPrice *big.Int) error { - txOpts.GasPrice = gasPrice + executeTransaction := func(bumpedGasPrices *big.Int) (*types.Transaction, error) { + txOpts.GasPrice = bumpedGasPrices w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) - return err - } - executeTransaction := func(gasPrice *big.Int) (*types.Transaction, error) { + if err != nil { + return nil, err + } + tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { // Retry with fallback @@ -111,7 +112,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return tx, err } - return utils.SendTransactionWithInfiniteRetryAndBumpingGasPrice(beforeTransaction, executeTransaction, w.Client, tx.GasPrice()) + return utils.SendTransactionWithInfiniteRetryAndBumpingGasPrice(executeTransaction, w.Client, tx.GasPrice()) } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 30755ce23..3d10ada2e 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -63,17 +63,12 @@ func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) // it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry` // and pass it to beforeTransaction and executeTransaction (make sure you update the txOpts with the new price) // This process happens indefinitely until we get the receipt or the receipt status is an err. -func SendTransactionWithInfiniteRetryAndBumpingGasPrice(beforeTransaction func(*big.Int) error, executeTransaction func(*big.Int) (*types.Transaction, error), client eth.InstrumentedClient, baseGasPrice *big.Int) (*types.Receipt, error) { +func SendTransactionWithInfiniteRetryAndBumpingGasPrice(executeTransaction func(*big.Int) (*types.Transaction, error), client eth.InstrumentedClient, baseGasPrice *big.Int) (*types.Receipt, error) { i := 0 sendTransaction := func() (*types.Receipt, error) { i++ gasPrice := CalculateGasPriceBumpBasedOnRetry(baseGasPrice, i) - err := beforeTransaction(gasPrice) - if err != nil { - return nil, err - } - tx, err := executeTransaction(gasPrice) if err != nil { return nil, err From bf7dad5f71b8f8de830afe6d754cb2981d2ca0a0 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 17:56:15 -0300 Subject: [PATCH 015/135] fix: unrecoverable errs --- core/chainio/avs_writer.go | 10 +++++++--- core/utils/eth_client_utils.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index b538d1d50..46042def7 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" + connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/config" "github.com/yetanotherco/aligned_layer/core/utils" ) @@ -100,16 +101,19 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { - return nil, err + return nil, connection.PermanentError{Inner: err} } tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { // Retry with fallback tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - return tx, err + if err != nil { + return nil, connection.PermanentError{Inner: err} + } + return tx, nil } - return tx, err + return tx, nil } return utils.SendTransactionWithInfiniteRetryAndBumpingGasPrice(executeTransaction, w.Client, tx.GasPrice()) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 3d10ada2e..eeb165afa 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -61,7 +61,7 @@ func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) // Sends a transaction and waits for the receipt for three blocks, if not received // it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry` -// and pass it to beforeTransaction and executeTransaction (make sure you update the txOpts with the new price) +// and pass it to executeTransaction (make sure you update the txOpts with the new gasPrice) // This process happens indefinitely until we get the receipt or the receipt status is an err. func SendTransactionWithInfiniteRetryAndBumpingGasPrice(executeTransaction func(*big.Int) (*types.Transaction, error), client eth.InstrumentedClient, baseGasPrice *big.Int) (*types.Receipt, error) { i := 0 From 72db7660bd9c9e0600eb850e6dbd2f7cecc170ea Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 17:57:46 -0300 Subject: [PATCH 016/135] feat: infinite retry for Retry without data --- core/connection.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/connection.go b/core/connection.go index b1a9f4f14..89fe34f53 100644 --- a/core/connection.go +++ b/core/connection.go @@ -69,7 +69,13 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) - maxRetriesBackoff := backoff.WithMaxRetries(expBackoff, maxTries) + var maxRetriesBackoff backoff.BackOff + + if maxTries > 0 { + maxRetriesBackoff = backoff.WithMaxRetries(expBackoff, maxTries) + } else { + maxRetriesBackoff = expBackoff + } return backoff.Retry(f, maxRetriesBackoff) } From ecaf600c7cc477ffe382965a7b7958361f18b14c Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 21 Oct 2024 17:57:46 -0300 Subject: [PATCH 017/135] feat: infinite retry for Retry without data --- core/connection.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/connection.go b/core/connection.go index b1a9f4f14..89fe34f53 100644 --- a/core/connection.go +++ b/core/connection.go @@ -69,7 +69,13 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) - maxRetriesBackoff := backoff.WithMaxRetries(expBackoff, maxTries) + var maxRetriesBackoff backoff.BackOff + + if maxTries > 0 { + maxRetriesBackoff = backoff.WithMaxRetries(expBackoff, maxTries) + } else { + maxRetriesBackoff = expBackoff + } return backoff.Retry(f, maxRetriesBackoff) } From 4c2f00d9d7640c2cbd6b2706d46ede2821a6d2f5 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 12:11:57 -0300 Subject: [PATCH 018/135] add retries to avs_subscriber --- core/chainio/avs_subscriber.go | 190 ++++++++++++++++++++++----------- 1 file changed, 126 insertions(+), 64 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 29afab7dc..16da4123f 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -4,9 +4,11 @@ import ( "context" "encoding/hex" "fmt" + "math/big" "sync" "time" + "github.com/ethereum/go-ethereum" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -17,6 +19,7 @@ import ( sdklogging "github.com/Layr-Labs/eigensdk-go/logging" "github.com/ethereum/go-ethereum/crypto" + connection "github.com/yetanotherco/aligned_layer/core" ) const ( @@ -62,18 +65,20 @@ func NewAvsSubscriberFromConfig(baseConfig *config.BaseConfig) (*AvsSubscriber, }, nil } -func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) (chan error, error) { +func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) (chan error, error) { // Create a new channel to receive new tasks internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) // Subscribe to new tasks - sub, err := subscribeToNewTasksV2(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + //TODO: Retry() + sub, err := subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err } - subFallback, err := subscribeToNewTasksV2(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + //TODO: Retry() + subFallback, err := subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err @@ -94,7 +99,8 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case newBatch := <-internalChannel: s.processNewBatchV2(newBatch, batchesSet, newBatchMutex, newTaskCreatedChan) case <-pollLatestBatchTicker.C: - latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV2() + //TODO: Retry() + latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV2Retryable() if err != nil { s.logger.Debug("Failed to get latest task from blockchain", "err", err) continue @@ -114,14 +120,16 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = subscribeToNewTasksV2(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + //TODO: Retry() + sub, err = subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = subscribeToNewTasksV2(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + //TODO: Retry() + subFallback, err = subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err } @@ -132,18 +140,20 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema return errorChannel, nil } -func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) (chan error, error) { +func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) (chan error, error) { // Create a new channel to receive new tasks internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) // Subscribe to new tasks - sub, err := subscribeToNewTasksV3(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + //TODO: Retry() + sub, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err } - subFallback, err := subscribeToNewTasksV3(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + //TODO: Retry() + subFallback, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err @@ -164,7 +174,8 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case newBatch := <-internalChannel: s.processNewBatchV3(newBatch, batchesSet, newBatchMutex, newTaskCreatedChan) case <-pollLatestBatchTicker.C: - latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV3() + //TODO: Retry() + latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV3Retryable() if err != nil { s.logger.Debug("Failed to get latest task from blockchain", "err", err) continue @@ -184,14 +195,16 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = subscribeToNewTasksV3(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + //TODO: Retry() + sub, err = subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = subscribeToNewTasksV3(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + //TODO: Retry() + subFallback, err = subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err } @@ -202,48 +215,43 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema return errorChannel, nil } -func subscribeToNewTasksV2( +func subscribeToNewTasksV2Retrayable( serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, logger sdklogging.Logger, ) (event.Subscription, error) { - for i := 0; i < MaxRetries; i++ { - sub, err := serviceManager.WatchNewBatchV2( - &bind.WatchOpts{}, newTaskCreatedChan, nil, - ) - if err != nil { - logger.Warn("Failed to subscribe to new AlignedLayer tasks", "err", err) - time.Sleep(RetryInterval) - continue - } - - logger.Info("Subscribed to new AlignedLayer tasks") - return sub, nil + //TODO: Retry() + subscribe_func := func() (*event.Subscription, error) { + sub, err := serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) + return &sub, err } - - return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + if err != nil { + // If the retries stop early we propogate the Permanent error to the caller. + // TODO: ask if this is the behavior we want. + return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + } + logger.Info("Subscribed to new AlignedLayer tasks") + return *sub, nil } -func subscribeToNewTasksV3( +func subscribeToNewTasksV3Retryable( serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, logger sdklogging.Logger, ) (event.Subscription, error) { - for i := 0; i < MaxRetries; i++ { - sub, err := serviceManager.WatchNewBatchV3( - &bind.WatchOpts{}, newTaskCreatedChan, nil, - ) - if err != nil { - logger.Warn("Failed to subscribe to new AlignedLayer tasks", "err", err) - time.Sleep(RetryInterval) - continue - } - - logger.Info("Subscribed to new AlignedLayer tasks") - return sub, nil + //TODO: Retry() + subscribe_func := func() (*event.Subscription, error) { + sub, err := serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) + return &sub, err } - - return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + if err != nil { + // If the retries stop early we propogate the Permanent error to the caller. + return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + } + logger.Info("Subscribed to new AlignedLayer tasks") + return *sub, nil } func (s *AvsSubscriber) processNewBatchV2(batch *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, batchesSet map[[32]byte]struct{}, newBatchMutex *sync.Mutex, newTaskCreatedChan chan<- *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) { @@ -299,10 +307,17 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL } // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. -func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { - latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) +func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { + + //TODO: Retry() + latestBlock_func := func() (*uint64, error) { + sub, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) + return &sub, err + } + latestBlock, err := connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) + //TODO: Retry() + latestBlock, err = connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, err } @@ -310,13 +325,18 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag var fromBlock uint64 - if latestBlock < BlockInterval { + if *latestBlock < BlockInterval { fromBlock = 0 } else { - fromBlock = latestBlock - BlockInterval + fromBlock = *latestBlock - BlockInterval } - logs, err := s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + //TODO: Retry() + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + } + //logs, err := s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + logs, err := connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, err } @@ -338,8 +358,17 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) + //TODO: Retry() + batchState_func := func() (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) + return &state, err + } + state, err := connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, err } @@ -352,10 +381,18 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag } // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. -func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { - latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) +func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { + //TODO: Retry() + latestBlock_func := func() (*uint64, error) { + latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) + return &latestBlock, err + } + //latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) + latestBlock, err := connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) + //TODO: Retry() + //s.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) + latestBlock, err = connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, err } @@ -363,13 +400,17 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag var fromBlock uint64 - if latestBlock < BlockInterval { + if *latestBlock < BlockInterval { fromBlock = 0 } else { - fromBlock = latestBlock - BlockInterval + fromBlock = *latestBlock - BlockInterval } - logs, err := s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + //TODO: Retry() + filterNewBatchV3_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { + return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + } + logs, err := connection.RetryWithData(filterNewBatchV3_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, err } @@ -391,8 +432,17 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) - + //TODO: Retry() + batchesState_func := func() (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) + return &state, err + } + //state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) + state, err := connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, err } @@ -404,22 +454,34 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag return lastLog, nil } -func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { - currentBlock, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) +func (s *AvsSubscriber) WaitForOneBlockRetryable(startBlock uint64) error { + //TODO: Retry() + blockNum_func := func() (*uint64, error) { + block_num, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) + return &block_num, err + } + currentBlock, err := connection.RetryWithData(blockNum_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // try with the fallback client - currentBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) + //TODO: Retry() + currentBlock, err = connection.RetryWithData(blockNum_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return err } } - if currentBlock <= startBlock { // should really be == but just in case + if *currentBlock <= startBlock { // should really be == but just in case // Subscribe to new head c := make(chan *types.Header) - sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(context.Background(), c) + //TODO: Retry() + subscribeNewHead_func := func() (*ethereum.Subscription, error) { + sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(context.Background(), c) + return &sub, err + } + sub, err := connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(context.Background(), c) + //TODO: Retry() + sub, err = connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return err } @@ -427,7 +489,7 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // Read channel for the new block <-c - sub.Unsubscribe() + (*sub).Unsubscribe() } return nil From 9e53e56661962436447ed67db124ef24257c507b Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 12:12:07 -0300 Subject: [PATCH 019/135] add retries to avs_writer --- core/chainio/avs_writer.go | 60 ++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index a6a820195..04fa85d52 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" + connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/config" ) @@ -73,16 +74,19 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*common.Hash, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction - tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + respondToTaskV2_func := func() (*types.Transaction, error) { + return w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + } + tx, err := connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // Retry with fallback - tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err = connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, fmt.Errorf("transaction simulation failed: %v", err) } } - err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) + err = w.checkRespondToTaskFeeLimitRetryable(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { return nil, err } @@ -90,10 +94,13 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // Send the transaction txOpts.NoSend = false txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit - tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + respondToTaskV2_func = func() (*types.Transaction, error) { + return w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + } + tx, err = connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // Retry with fallback - tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err = connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { return nil, err } @@ -104,22 +111,31 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return &txHash, nil } -func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { +func (w *AvsWriter) checkRespondToTaskFeeLimitRetryable(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { aggregatorAddress := txOpts.From simulatedCost := new(big.Int).Mul(new(big.Int).SetUint64(tx.Gas()), tx.GasPrice()) w.logger.Info("Simulated cost", "cost", simulatedCost) // Get RespondToTaskFeeLimit - batchState, err := w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, batchIdentifierHash) + //TODO: make this a public type + batchesState_func := func() (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err := w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, batchIdentifierHash) + return &state, err + } + batchState, err := connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // Retry with fallback - batchState, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, batchIdentifierHash) + batchState, err = connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // Fallback also failed // Proceed to check values against simulated costs w.logger.Error("Failed to get batch state", "error", err) w.logger.Info("Proceeding with simulated cost checks") - return w.compareBalances(simulatedCost, aggregatorAddress, senderAddress) + return w.compareBalancesRetryable(simulatedCost, aggregatorAddress, senderAddress) } } // At this point, batchState was successfully retrieved @@ -131,26 +147,29 @@ func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bin return fmt.Errorf("cost of transaction is higher than Batch.RespondToTaskFeeLimit") } - return w.compareBalances(respondToTaskFeeLimit, aggregatorAddress, senderAddress) + return w.compareBalancesRetryable(respondToTaskFeeLimit, aggregatorAddress, senderAddress) } -func (w *AvsWriter) compareBalances(amount *big.Int, aggregatorAddress common.Address, senderAddress [20]byte) error { - if err := w.compareAggregatorBalance(amount, aggregatorAddress); err != nil { +func (w *AvsWriter) compareBalancesRetryable(amount *big.Int, aggregatorAddress common.Address, senderAddress [20]byte) error { + if err := w.compareAggregatorBalanceRetryable(amount, aggregatorAddress); err != nil { return err } - if err := w.compareBatcherBalance(amount, senderAddress); err != nil { + if err := w.compareBatcherBalanceRetryable(amount, senderAddress); err != nil { return err } return nil } -func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress common.Address) error { +func (w *AvsWriter) compareAggregatorBalanceRetryable(amount *big.Int, aggregatorAddress common.Address) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // Get Agg wallet balance - aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, nil) + balanceAt_func := func() (*big.Int, error) { + return w.Client.BalanceAt(ctx, aggregatorAddress, nil) + } + aggregatorBalance, err := connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, nil) + aggregatorBalance, err = connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // Ignore and continue. w.logger.Error("failed to get aggregator balance: %v", err) @@ -164,11 +183,14 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress return nil } -func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byte) error { +func (w *AvsWriter) compareBatcherBalanceRetryable(amount *big.Int, senderAddress [20]byte) error { // Get batcher balance - batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + batcherBalances_func := func() (*big.Int, error) { + return w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + } + batcherBalance, err := connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) + batcherBalance, err = connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // Ignore and continue. w.logger.Error("Failed to get batcherBalance", "error", err) From 27c8cc6aba11b7110d055dabf53df816dba66881 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 12:12:49 -0300 Subject: [PATCH 020/135] add start to basic retry test in aggregator --- core/connection_test.go | 258 ++++++++++++++++++++++++++++++++- core/utils/eth_client_utils.go | 13 ++ 2 files changed, 270 insertions(+), 1 deletion(-) diff --git a/core/connection_test.go b/core/connection_test.go index a922ec10e..92fe44d2b 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -1,14 +1,30 @@ package connection_test import ( + "bytes" + "context" "fmt" + "log" + "math/big" + "os" + "os/exec" + "strconv" + "syscall" "testing" + "time" + "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" + rpccalls "github.com/Layr-Labs/eigensdk-go/metrics/collectors/rpc_calls" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" connection "github.com/yetanotherco/aligned_layer/core" + "github.com/yetanotherco/aligned_layer/core/utils" ) func DummyFunction(x uint64) (uint64, error) { - fmt.Println("Doing some work...") + fmt.Println("Starting Anvil on Port ") if x == 42 { return 0, connection.PermanentError{Inner: fmt.Errorf("Permanent error!")} } else if x < 42 { @@ -40,3 +56,243 @@ func TestRetry(t *testing.T) { t.Errorf("Retry error!: %s", err) } } + +/* +Starts an anvil instance via the cli. +Assumes that anvil is installed but checks. +*/ +func SetupAnvil(port uint16) (*exec.Cmd, *eth.InstrumentedClient, error) { + + path, err := exec.LookPath("anvil") + if err != nil { + fmt.Printf("Could not find `anvil` executable in `%s`\n", path) + } + + port_str := strconv.Itoa(int(port)) + http_rpc_url := fmt.Sprintf("http://localhost:%d", port) + + // Create a command + cmd := exec.Command("anvil", "--port", port_str, "--load-state", "../contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json", "--block-time", "7") + + // Prepare to capture the output + var out bytes.Buffer + cmd.Stdout = &out + + // Run the command + err = cmd.Start() + if err != nil { + fmt.Printf("Error: %s\n", err) + } + + // Delay needed for anvil to start + time.Sleep(1 * time.Second) + + reg := prometheus.NewRegistry() + rpcCallsCollector := rpccalls.NewCollector("ethRpc", reg) + ethRpcClient, err := eth.NewInstrumentedClient(http_rpc_url, rpcCallsCollector) + if err != nil { + log.Fatal("Error initializing eth rpc client: ", err) + } + + return cmd, ethRpcClient, nil +} + +func TestAnvilSetupKill(t *testing.T) { + // Start Anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + log.Fatal("Error setting up Anvil: ", err) + } + + // Get Anvil PID + pid := cmd.Process.Pid + p, err := os.FindProcess(pid) + if err != nil { + log.Fatal("Error finding Anvil Process: ", err) + } + err = p.Signal(syscall.Signal(0)) + assert.Nil(t, err, "Anvil Process Killed") + + // Kill Anvil + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("Error killing process: %v\n", err) + return + } + + // Check that PID is not currently present in running processes. + // FindProcess always succeeds so on Unix systems we call it below. + p, err = os.FindProcess(pid) + if err != nil { + log.Fatal("Error finding Anvil Process: ", err) + } + // Ensure process has exited + err = p.Signal(syscall.Signal(0)) + + assert.Nil(t, err, "Anvil Process Killed") +} + +// Waits for receipt from anvil node -> Will fail to get receipt +func TestWaitForTransactionReceiptRetryable(t *testing.T) { + // Start anvil + cmd, client, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + tx := types.NewTx(&types.AccessListTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }) + + ctx := context.WithoutCancel(context.Background()) + + hash := tx.Hash() + + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + if err != nil { + t.Errorf("Retry error!: %s", err) + } else { + fmt.Printf("Tx Hash: %s\n", receipt.TxHash) + } + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("Error killing process: %v\n", err) + return + } +} + +func TestRespondToTaskV2(t *testing.T) { + // Start anvil + cmd, client, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + tx := types.NewTx(&types.AccessListTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }) + + ctx := context.WithoutCancel(context.Background()) + + hash := tx.Hash() + + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + if err != nil { + t.Errorf("Retry error!: %s", err) + } else { + fmt.Printf("Tx Hash: %s\n", receipt.TxHash) + } + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("Error killing process: %v\n", err) + return + } +} + +func TestRespondToTaskV3(t *testing.T) { + // Start anvil + cmd, client, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + tx := types.NewTx(&types.AccessListTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }) + + ctx := context.WithoutCancel(context.Background()) + + hash := tx.Hash() + + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + if err != nil { + t.Errorf("Retry error!: %s", err) + } else { + fmt.Printf("Tx Hash: %s\n", receipt.TxHash) + } + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("Error killing process: %v\n", err) + return + } +} + +func SubscribeToNewTasksV2(t *testing.T) { + // Start anvil + cmd, client, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + tx := types.NewTx(&types.AccessListTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }) + + ctx := context.WithoutCancel(context.Background()) + + hash := tx.Hash() + + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + if err != nil { + t.Errorf("Retry error!: %s", err) + } else { + fmt.Printf("Tx Hash: %s\n", receipt.TxHash) + } + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("Error killing process: %v\n", err) + return + } +} + +func SubscribeToNewTasksV3(t *testing.T) { + // Start anvil + cmd, client, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + tx := types.NewTx(&types.AccessListTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }) + + ctx := context.WithoutCancel(context.Background()) + + hash := tx.Hash() + + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + if err != nil { + t.Errorf("Retry error!: %s", err) + } else { + fmt.Printf("Tx Hash: %s\n", receipt.TxHash) + } + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("Error killing process: %v\n", err) + return + } +} diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index afa26b867..8716fcb8a 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -10,6 +10,7 @@ import ( eigentypes "github.com/Layr-Labs/eigensdk-go/types" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + connection "github.com/yetanotherco/aligned_layer/core" ) const maxRetries = 25 @@ -27,6 +28,18 @@ func WaitForTransactionReceipt(client eth.InstrumentedClient, ctx context.Contex return nil, fmt.Errorf("transaction receipt not found for txHash: %s", txHash.String()) } +func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { + receipt_func := func() (*types.Receipt, error) { return client.TransactionReceipt(ctx, txHash) } + //TODO: make these constants some value. + receipt, err := connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + if err != nil { + // If the retries stop early we propogate the Permanent error to the caller. + // TODO: ask if this is the behavior we want. + return nil, err + } + return receipt, nil +} + func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { quorumNums := make(eigentypes.QuorumNums, len(quorumNumbersBytes)) for i, quorumNumberByte := range quorumNumbersBytes { From 5fa0a2852b57c60675047dbd26bdce4f4f4a7155 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 12:12:57 -0300 Subject: [PATCH 021/135] add retries to aggregator --- aggregator/internal/pkg/aggregator.go | 30 ++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index f17591edc..9f514d090 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -10,6 +10,7 @@ import ( gethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" + connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/metrics" sdkclients "github.com/Layr-Labs/eigensdk-go/chainio/clients" @@ -262,24 +263,20 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:]), "taskCreatedBlock", taskCreatedBlock) - err := agg.avsSubscriber.WaitForOneBlock(taskCreatedBlock) + err := agg.avsSubscriber.WaitForOneBlockRetryable(taskCreatedBlock) if err != nil { agg.logger.Error("Error waiting for one block, sending anyway", "err", err) } agg.logger.Info("Sending aggregated response onchain", "taskIndex", blsAggServiceResp.TaskIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - for i := 0; i < MaxSentTxRetries; i++ { - _, err = agg.sendAggregatedResponse(batchIdentifierHash, batchData.BatchMerkleRoot, batchData.SenderAddress, nonSignerStakesAndSignature) - if err == nil { - agg.logger.Info("Aggregator successfully responded to task", - "taskIndex", blsAggServiceResp.TaskIndex, - "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - return - } - - // Sleep for a bit before retrying - time.Sleep(2 * time.Second) + //TODO: Retry() + _, err = agg.sendAggregatedResponse(batchIdentifierHash, batchData.BatchMerkleRoot, batchData.SenderAddress, nonSignerStakesAndSignature) + if err == nil { + agg.logger.Info("Aggregator successfully responded to task", + "taskIndex", blsAggServiceResp.TaskIndex, + "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) + return } agg.logger.Error("Aggregator failed to respond to task, this batch will be lost", @@ -291,6 +288,8 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.telemetry.LogTaskError(batchData.BatchMerkleRoot, err) } +// TODO: should we assume this is idempotent + // / Sends response to contract and waits for transaction receipt // / Returns error if it fails to send tx or receipt is not found func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*gethtypes.Receipt, error) { @@ -312,7 +311,7 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Sending aggregated response for batch %s", hex.EncodeToString(batchIdentifierHash[:])) - receipt, err := utils.WaitForTransactionReceipt( + receipt, err := utils.WaitForTransactionReceiptRetryable( agg.AggregatorConfig.BaseConfig.EthRpcClient, context.Background(), *txHash) if err != nil { agg.telemetry.LogTaskError(batchMerkleRoot, err) @@ -366,7 +365,10 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(QUORUM_NUMBER)} quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(QUORUM_THRESHOLD)} - err := agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + initilizeNewTask_func := func() error { + return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + } + err := connection.Retry(initilizeNewTask_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) // FIXME(marian): When this errors, should we retry initializing new task? Logging fatal for now. if err != nil { agg.logger.Fatalf("BLS aggregation service error when initializing new task: %s", err) From 2ec553ba74094935d220ff1de08d7fc283bfd006 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 12:13:17 -0300 Subject: [PATCH 022/135] add constants to connections --- core/connection.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/connection.go b/core/connection.go index 89fe34f53..f023d382f 100644 --- a/core/connection.go +++ b/core/connection.go @@ -19,6 +19,13 @@ func (e PermanentError) Is(err error) bool { return ok } +const MinDelay = 1000 +const RetryFactor = 2 +const NumRetries = 3 + +//NOTE: I reverted the change Marcos made to make the return type a *T as some return types of the function calls. +// were not pointers or uint64. + // Same as Retry only that the functionToRetry can return a value upon correct execution func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { i := 0 From 34a80cc69894915bb53fbeaf760727d7597020e0 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 12:14:48 -0300 Subject: [PATCH 023/135] Change subscriber function call --- aggregator/internal/pkg/subscriber.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/internal/pkg/subscriber.go b/aggregator/internal/pkg/subscriber.go index 4aef715da..b592f6ace 100644 --- a/aggregator/internal/pkg/subscriber.go +++ b/aggregator/internal/pkg/subscriber.go @@ -24,7 +24,7 @@ func (agg *Aggregator) SubscribeToNewTasks() error { func (agg *Aggregator) subscribeToNewTasks() error { var err error - agg.taskSubscriber, err = agg.avsSubscriber.SubscribeToNewTasksV3(agg.NewBatchChan) + agg.taskSubscriber, err = agg.avsSubscriber.SubscribeToNewTasksV3Retryable(agg.NewBatchChan) if err != nil { agg.AggregatorConfig.BaseConfig.Logger.Info("Failed to create task subscriber", "err", err) From 99aa09475a470987dfe9b1ccbb59364ac4fd8133 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 12:58:46 -0300 Subject: [PATCH 024/135] refactor: clearer math in bump calculation --- core/utils/eth_client_utils.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index eeb165afa..c4f592fcd 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -52,11 +52,11 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e // Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice and retry iteration. // It adds a i/10 percentage to the current prices, where i represents the iteration. func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) *big.Int { - factor := (new(big.Int).Add(big.NewInt(100), new(big.Int).Mul(big.NewInt(int64(iteration)), big.NewInt(10)))) - gasPrice := new(big.Int).Mul(currentGasPrice, factor) - gasPrice = gasPrice.Div(gasPrice, big.NewInt(100)) + percentageBump := new(big.Int).Div(big.NewInt(int64(iteration)), big.NewInt(10)) + bumpAmount := new(big.Int).Mul(currentGasPrice, percentageBump) + bumpedGasPrice := new(big.Int).Add(currentGasPrice, bumpAmount) - return gasPrice + return bumpedGasPrice } // Sends a transaction and waits for the receipt for three blocks, if not received From b614916c632f377f4cc258d7107004d290f9ffd9 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 12:59:41 -0300 Subject: [PATCH 025/135] refactor: retry remove i --- core/connection.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/connection.go b/core/connection.go index 89fe34f53..0322d6888 100644 --- a/core/connection.go +++ b/core/connection.go @@ -21,10 +21,8 @@ func (e PermanentError) Is(err error) bool { // Same as Retry only that the functionToRetry can return a value upon correct execution func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { - i := 0 f := func() (*T, error) { val, err := functionToRetry() - i++ if perm, ok := err.(PermanentError); err != nil && ok { return nil, backoff.Permanent(perm.Inner) } @@ -54,10 +52,8 @@ func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, f // The function to be retried should return `PermanentError` when the condition for stop retrying // is met. func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64) error { - i := 0 f := func() error { err := functionToRetry() - i++ if perm, ok := err.(PermanentError); err != nil && ok { return backoff.Permanent(perm.Inner) } From 527489b78868f2d8c4347ba3c77f91e7dcf4ca75 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 14:31:59 -0300 Subject: [PATCH 026/135] refactor: remove SendTransactionWithInfiniteRetryAndBumpingGasPrice --- core/chainio/avs_writer.go | 29 ++++++++++++++++++++++++----- core/utils/eth_client_utils.go | 34 ---------------------------------- 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 46042def7..0d81083d1 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -95,8 +95,15 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts.NoSend = false txOpts.Nonce = txNonce - executeTransaction := func(bumpedGasPrices *big.Int) (*types.Transaction, error) { - txOpts.GasPrice = bumpedGasPrices + // Sends a transaction and waits for the receipt for three blocks, if not received + // it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry` + // This process happens indefinitely until we sendTransaction does not return err. + i := 0 + sendTransaction := func() (*types.Receipt, error) { + i++ + gasPrice := utils.CalculateGasPriceBumpBasedOnRetry(tx.GasPrice(), i) + txOpts.GasPrice = gasPrice + w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) @@ -111,12 +118,24 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe if err != nil { return nil, connection.PermanentError{Inner: err} } - return tx, nil } - return tx, nil + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*36) + defer cancel() + receipt, err := utils.WaitForTransactionReceipt(w.Client, ctx, tx.Hash()) + + if receipt != nil { + return receipt, nil + } + // if we are here, this means we have reached the timeout (after three blocks it hasn't been included) + // so we try again by bumping the fee to make sure its included + if err != nil { + return nil, err + } + return nil, fmt.Errorf("transaction failed") } - return utils.SendTransactionWithInfiniteRetryAndBumpingGasPrice(executeTransaction, w.Client, tx.GasPrice()) + return connection.RetryWithData(sendTransaction, 1000, 2, 0) } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index c4f592fcd..38bba770c 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -11,7 +11,6 @@ import ( eigentypes "github.com/Layr-Labs/eigensdk-go/types" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - connection "github.com/yetanotherco/aligned_layer/core" ) const maxRetries = 25 @@ -58,36 +57,3 @@ func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) return bumpedGasPrice } - -// Sends a transaction and waits for the receipt for three blocks, if not received -// it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry` -// and pass it to executeTransaction (make sure you update the txOpts with the new gasPrice) -// This process happens indefinitely until we get the receipt or the receipt status is an err. -func SendTransactionWithInfiniteRetryAndBumpingGasPrice(executeTransaction func(*big.Int) (*types.Transaction, error), client eth.InstrumentedClient, baseGasPrice *big.Int) (*types.Receipt, error) { - i := 0 - sendTransaction := func() (*types.Receipt, error) { - i++ - gasPrice := CalculateGasPriceBumpBasedOnRetry(baseGasPrice, i) - - tx, err := executeTransaction(gasPrice) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*36) - defer cancel() - receipt, err := WaitForTransactionReceipt(client, ctx, tx.Hash()) - - if receipt != nil { - return receipt, nil - } - // if we are here, this means we have reached the timeout (after three blocks it hasn't been included) - // so we try again by bumping the fee to make sure its included - if err != nil { - return nil, err - } - return nil, fmt.Errorf("transaction failed") - - } - return connection.RetryWithData(sendTransaction, 1000, 2, 0) -} From 5abc31e9acaa6f379cbddb6545f7a0ac781af714 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 15:29:36 -0300 Subject: [PATCH 027/135] refactor: gas price bump based on last bump --- core/chainio/avs_writer.go | 12 ++++++++---- core/utils/eth_client_utils.go | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 0d81083d1..6533dbde7 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -20,6 +20,10 @@ import ( "github.com/yetanotherco/aligned_layer/core/utils" ) +const ( + gasBumpPercentage int = 10 +) + type AvsWriter struct { *avsregistry.ChainWriter AvsContractBindings *AvsServiceBindings @@ -98,11 +102,11 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // Sends a transaction and waits for the receipt for three blocks, if not received // it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry` // This process happens indefinitely until we sendTransaction does not return err. - i := 0 + lastTxGasPrice := tx.GasPrice() sendTransaction := func() (*types.Receipt, error) { - i++ - gasPrice := utils.CalculateGasPriceBumpBasedOnRetry(tx.GasPrice(), i) - txOpts.GasPrice = gasPrice + bumpedGasPrice := utils.CalculateGasPriceBump(lastTxGasPrice, gasBumpPercentage) + lastTxGasPrice = bumpedGasPrice + txOpts.GasPrice = bumpedGasPrice w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 38bba770c..eb87a69de 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -50,8 +50,8 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e // Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice and retry iteration. // It adds a i/10 percentage to the current prices, where i represents the iteration. -func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, iteration int) *big.Int { - percentageBump := new(big.Int).Div(big.NewInt(int64(iteration)), big.NewInt(10)) +func CalculateGasPriceBump(currentGasPrice *big.Int, percentage int) *big.Int { + percentageBump := new(big.Int).Div(big.NewInt(int64(percentage)), big.NewInt(100)) bumpAmount := new(big.Int).Mul(currentGasPrice, percentageBump) bumpedGasPrice := new(big.Int).Add(currentGasPrice, bumpAmount) From 6f85dda15cfb4e93d195852f179f67f14e5d655b Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 16:01:25 -0300 Subject: [PATCH 028/135] docs: SendAggregatedResponse and gas price calc --- core/chainio/avs_writer.go | 8 ++++---- core/utils/eth_client_utils.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 6533dbde7..da1554751 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -76,7 +76,10 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E }, nil } -func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Receipt, error) { +// Sends AggregatedResponse and waits for the receipt for three blocks, if not received +// it will try again bumping the last tx gas price based on `CalculateGasPriceBump` +// This process happens indefinitely until the transaction is included. +func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, onRetry func()) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) @@ -99,9 +102,6 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts.NoSend = false txOpts.Nonce = txNonce - // Sends a transaction and waits for the receipt for three blocks, if not received - // it will try again bumping the gas price based on `CalculateGasPriceBumpBasedOnRetry` - // This process happens indefinitely until we sendTransaction does not return err. lastTxGasPrice := tx.GasPrice() sendTransaction := func() (*types.Receipt, error) { bumpedGasPrice := utils.CalculateGasPriceBump(lastTxGasPrice, gasBumpPercentage) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index eb87a69de..08cbc5233 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -48,8 +48,8 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e return quorumThresholdPercentages } -// Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice and retry iteration. -// It adds a i/10 percentage to the current prices, where i represents the iteration. +// Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice and percentage. +// It adds a the percentage to the current gas price. func CalculateGasPriceBump(currentGasPrice *big.Int, percentage int) *big.Int { percentageBump := new(big.Int).Div(big.NewInt(int64(percentage)), big.NewInt(100)) bumpAmount := new(big.Int).Mul(currentGasPrice, percentageBump) From 96f2ace6b3b658e0b9b4a3f780f27f475357cfef Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 16:06:22 -0300 Subject: [PATCH 029/135] feat: metrics for bumped gas price --- aggregator/internal/pkg/aggregator.go | 3 ++- core/chainio/avs_writer.go | 6 ++++++ metrics/metrics.go | 20 +++++++++++++++----- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 6568d2106..71ab6540f 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -301,7 +301,8 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "senderAddress", hex.EncodeToString(senderAddress[:]), "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) - receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + onRetry := func() { agg.metrics.IncBumpedGasPriceForAggregatedResponse() } + receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature, onRetry) if err != nil { agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Error sending aggregated response for batch %s. Error: %s", hex.EncodeToString(batchIdentifierHash[:]), err) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index da1554751..d025a9ae1 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -103,7 +103,13 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts.Nonce = txNonce lastTxGasPrice := tx.GasPrice() + i := 0 sendTransaction := func() (*types.Receipt, error) { + if i > 0 { + onRetry() + } + i++ + bumpedGasPrice := utils.CalculateGasPriceBump(lastTxGasPrice, gasBumpPercentage) lastTxGasPrice = bumpedGasPrice txOpts.GasPrice = bumpedGasPrice diff --git a/metrics/metrics.go b/metrics/metrics.go index 11c0829c4..efe5f3135 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -12,11 +12,12 @@ import ( ) type Metrics struct { - ipPortAddress string - logger logging.Logger - numAggregatedResponses prometheus.Counter - numAggregatorReceivedTasks prometheus.Counter - numOperatorTaskResponses prometheus.Counter + ipPortAddress string + logger logging.Logger + numAggregatedResponses prometheus.Counter + numAggregatorReceivedTasks prometheus.Counter + numOperatorTaskResponses prometheus.Counter + numBumpedGasPriceForAggregatedResponse prometheus.Counter } const alignedNamespace = "aligned" @@ -40,6 +41,11 @@ func NewMetrics(ipPortAddress string, reg prometheus.Registerer, logger logging. Name: "aggregator_received_tasks", Help: "Number of tasks received by the Service Manager", }), + numBumpedGasPriceForAggregatedResponse: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Namespace: alignedNamespace, + Name: "respond_to_task_gas_price_bumped", + Help: "Number of times gas price was bumped while sending aggregated response", + }), } } @@ -74,3 +80,7 @@ func (m *Metrics) IncAggregatedResponses() { func (m *Metrics) IncOperatorTaskResponses() { m.numOperatorTaskResponses.Inc() } + +func (m *Metrics) IncBumpedGasPriceForAggregatedResponse() { + m.numBumpedGasPriceForAggregatedResponse.Inc() +} From 34691018cab55ff2919eabbb7ad2e40f7650077a Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 16:23:28 -0300 Subject: [PATCH 030/135] chore: grafana view for metrics --- .../aligned/aggregator_batcher.json | 112 +++++++++++++++++- 1 file changed, 106 insertions(+), 6 deletions(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index 7f7c765ef..56d61ff33 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -758,7 +758,7 @@ "h": 7, "w": 10, "x": 0, - "y": 34 + "y": 20 }, "id": 9, "options": { @@ -910,7 +910,7 @@ "h": 7, "w": 10, "x": 10, - "y": 34 + "y": 20 }, "id": 1, "options": { @@ -974,7 +974,7 @@ "h": 7, "w": 5, "x": 0, - "y": 41 + "y": 27 }, "id": 8, "options": { @@ -1044,7 +1044,7 @@ "h": 7, "w": 5, "x": 5, - "y": 41 + "y": 27 }, "id": 7, "options": { @@ -1083,6 +1083,106 @@ "title": "Tasks Received", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of times gas price was bumped while sending an aggregated response.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 10, + "y": 27 + }, + "id": 20, + "interval": "36", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aligned_respond_to_task_gas_price_bumped", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Bumped gas price for aggregated responses", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -1109,7 +1209,7 @@ "h": 7, "w": 5, "x": 0, - "y": 48 + "y": 34 }, "id": 2, "options": { @@ -1179,7 +1279,7 @@ "h": 7, "w": 5, "x": 5, - "y": 48 + "y": 34 }, "id": 5, "options": { From 001eeab4e703be00bc76b840e1e5cee24cdd94cf Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 16:38:29 -0300 Subject: [PATCH 031/135] add retryable functions to operator --- operator/pkg/operator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/pkg/operator.go b/operator/pkg/operator.go index 8a96b51c4..eaf9eed2b 100644 --- a/operator/pkg/operator.go +++ b/operator/pkg/operator.go @@ -153,11 +153,11 @@ func NewOperatorFromConfig(configuration config.OperatorConfig) (*Operator, erro } func (o *Operator) SubscribeToNewTasksV2() (chan error, error) { - return o.avsSubscriber.SubscribeToNewTasksV2(o.NewTaskCreatedChanV2) + return o.avsSubscriber.SubscribeToNewTasksV2Retrayable(o.NewTaskCreatedChanV2) } func (o *Operator) SubscribeToNewTasksV3() (chan error, error) { - return o.avsSubscriber.SubscribeToNewTasksV3(o.NewTaskCreatedChanV3) + return o.avsSubscriber.SubscribeToNewTasksV3Retryable(o.NewTaskCreatedChanV3) } type OperatorLastProcessedBatch struct { From 04375dd8e691504c96e94f803fb449626332ce55 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 16:38:41 -0300 Subject: [PATCH 032/135] refactor connection_test --- core/connection_test.go | 139 +++++----------------------------------- 1 file changed, 16 insertions(+), 123 deletions(-) diff --git a/core/connection_test.go b/core/connection_test.go index 92fe44d2b..18e58b59f 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -133,12 +133,8 @@ func TestAnvilSetupKill(t *testing.T) { // Waits for receipt from anvil node -> Will fail to get receipt func TestWaitForTransactionReceiptRetryable(t *testing.T) { - // Start anvil - cmd, client, err := SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } + // Retry call Params to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") tx := types.NewTx(&types.AccessListTx{ Nonce: 1, @@ -151,148 +147,45 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { ctx := context.WithoutCancel(context.Background()) hash := tx.Hash() - - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - if err != nil { - t.Errorf("Retry error!: %s", err) - } else { - fmt.Printf("Tx Hash: %s\n", receipt.TxHash) - } - - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) - return - } -} - -func TestRespondToTaskV2(t *testing.T) { // Start anvil cmd, client, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } - to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") - tx := types.NewTx(&types.AccessListTx{ - Nonce: 1, - GasPrice: big.NewInt(500), - Gas: 1000000, - To: &to, - Value: big.NewInt(1), - }) - - ctx := context.WithoutCancel(context.Background()) - - hash := tx.Hash() - - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - if err != nil { - t.Errorf("Retry error!: %s", err) - } else { - fmt.Printf("Tx Hash: %s\n", receipt.TxHash) - } + // Assert Call succeeds why Anvil running + _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + assert.NotNil(t, err, "Call to Anvil failed") + // Kill Anvil if err := cmd.Process.Kill(); err != nil { fmt.Printf("Error killing process: %v\n", err) return } -} - -func TestRespondToTaskV3(t *testing.T) { - // Start anvil - cmd, client, err := SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } - - to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") - tx := types.NewTx(&types.AccessListTx{ - Nonce: 1, - GasPrice: big.NewInt(500), - Gas: 1000000, - To: &to, - Value: big.NewInt(1), - }) - ctx := context.WithoutCancel(context.Background()) - - hash := tx.Hash() - - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + // Check that permanent error thrown + _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) if err != nil { t.Errorf("Retry error!: %s", err) - } else { - fmt.Printf("Tx Hash: %s\n", receipt.TxHash) - } - - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) - return } -} -func SubscribeToNewTasksV2(t *testing.T) { // Start anvil - cmd, client, err := SetupAnvil(8545) + _, client, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } + _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + assert.NotNil(t, err, "Call to Anvil failed") +} - to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") - tx := types.NewTx(&types.AccessListTx{ - Nonce: 1, - GasPrice: big.NewInt(500), - Gas: 1000000, - To: &to, - Value: big.NewInt(1), - }) - - ctx := context.WithoutCancel(context.Background()) - - hash := tx.Hash() +func TestRespondToTaskV2(t *testing.T) { +} - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - if err != nil { - t.Errorf("Retry error!: %s", err) - } else { - fmt.Printf("Tx Hash: %s\n", receipt.TxHash) - } +func TestRespondToTaskV3(t *testing.T) { +} - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) - return - } +func SubscribeToNewTasksV2(t *testing.T) { } func SubscribeToNewTasksV3(t *testing.T) { - // Start anvil - cmd, client, err := SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } - - to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") - tx := types.NewTx(&types.AccessListTx{ - Nonce: 1, - GasPrice: big.NewInt(500), - Gas: 1000000, - To: &to, - Value: big.NewInt(1), - }) - - ctx := context.WithoutCancel(context.Background()) - - hash := tx.Hash() - - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - if err != nil { - t.Errorf("Retry error!: %s", err) - } else { - fmt.Printf("Tx Hash: %s\n", receipt.TxHash) - } - - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) - return - } } From 7d8660b6b19662b40bd14e2a0cd83bba6ba08d45 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 22 Oct 2024 17:13:13 -0300 Subject: [PATCH 033/135] fix: gas price bump calculation --- core/utils/eth_client_utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 08cbc5233..3c23bd900 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -51,8 +51,9 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e // Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice and percentage. // It adds a the percentage to the current gas price. func CalculateGasPriceBump(currentGasPrice *big.Int, percentage int) *big.Int { - percentageBump := new(big.Int).Div(big.NewInt(int64(percentage)), big.NewInt(100)) + percentageBump := big.NewInt(int64(percentage)) bumpAmount := new(big.Int).Mul(currentGasPrice, percentageBump) + bumpAmount = new(big.Int).Div(bumpAmount, big.NewInt(100)) bumpedGasPrice := new(big.Int).Add(currentGasPrice, bumpAmount) return bumpedGasPrice From 529fbe475ab75a3b9e378efedf101a085832bc33 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 22 Oct 2024 23:04:31 -0300 Subject: [PATCH 034/135] rm old waitForTransaction Function --- core/utils/eth_client_utils.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 8716fcb8a..4cef9d7b5 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -2,9 +2,6 @@ package utils import ( "context" - "time" - - "fmt" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" eigentypes "github.com/Layr-Labs/eigensdk-go/types" @@ -13,28 +10,10 @@ import ( connection "github.com/yetanotherco/aligned_layer/core" ) -const maxRetries = 25 -const sleepTime = 5 * time.Second - -func WaitForTransactionReceipt(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { - for i := 0; i < maxRetries; i++ { - receipt, err := client.TransactionReceipt(ctx, txHash) - if err != nil { - time.Sleep(sleepTime) - } else { - return receipt, nil - } - } - return nil, fmt.Errorf("transaction receipt not found for txHash: %s", txHash.String()) -} - func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { receipt_func := func() (*types.Receipt, error) { return client.TransactionReceipt(ctx, txHash) } - //TODO: make these constants some value. receipt, err := connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - // If the retries stop early we propogate the Permanent error to the caller. - // TODO: ask if this is the behavior we want. return nil, err } return receipt, nil From 30a053d73d52b80b3988d1394ad6da64b7cfc4bb Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 09:48:13 -0300 Subject: [PATCH 035/135] feat: new gas price algorithm for respondtotask --- core/chainio/avs_writer.go | 17 ++++++++++++----- core/utils/eth_client_utils.go | 9 +++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index d025a9ae1..2cb068aaf 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -21,7 +21,7 @@ import ( ) const ( - gasBumpPercentage int = 10 + gasBumpPercentage int = 20 ) type AvsWriter struct { @@ -79,6 +79,7 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E // Sends AggregatedResponse and waits for the receipt for three blocks, if not received // it will try again bumping the last tx gas price based on `CalculateGasPriceBump` // This process happens indefinitely until the transaction is included. +// Note: If the rpc endpoints fail, the retry will stop, as it will infinitely try func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, onRetry func()) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction @@ -102,7 +103,6 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts.NoSend = false txOpts.Nonce = txNonce - lastTxGasPrice := tx.GasPrice() i := 0 sendTransaction := func() (*types.Receipt, error) { if i > 0 { @@ -110,9 +110,16 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe } i++ - bumpedGasPrice := utils.CalculateGasPriceBump(lastTxGasPrice, gasBumpPercentage) - lastTxGasPrice = bumpedGasPrice - txOpts.GasPrice = bumpedGasPrice + // get gas price + gasPrice, err := w.Client.SuggestGasPrice(context.Background()) + if err != nil { + // Retry with fallback + gasPrice, err = w.ClientFallback.SuggestGasPrice(context.Background()) + if err != nil { + return nil, fmt.Errorf("transaction simulation failed: %v", err) + } + } + txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, gasBumpPercentage, i) w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 3c23bd900..08dd4c911 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -48,10 +48,11 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e return quorumThresholdPercentages } -// Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice and percentage. -// It adds a the percentage to the current gas price. -func CalculateGasPriceBump(currentGasPrice *big.Int, percentage int) *big.Int { - percentageBump := big.NewInt(int64(percentage)) +// Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice a constant percentage and the retry number. +// It adds a the percentage to the current gas price and a 5% * i, where i is the iteration number. That is: +func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, percentage int, i int) *big.Int { + retryPercentage := new(big.Int).Mul(big.NewInt(5), big.NewInt(int64(i))) + percentageBump := new(big.Int).Add(big.NewInt(int64(percentage)), retryPercentage) bumpAmount := new(big.Int).Mul(currentGasPrice, percentageBump) bumpAmount = new(big.Int).Div(bumpAmount, big.NewInt(100)) bumpedGasPrice := new(big.Int).Add(currentGasPrice, bumpAmount) From 21b5e2a762316d68580ded3e359254e7122b695f Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 10:00:53 -0300 Subject: [PATCH 036/135] chore: update metrics json --- .../aligned/aggregator_batcher.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index 23c678a77..614dac92b 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -850,7 +850,7 @@ "h": 7, "w": 10, "x": 0, - "y": 42 + "y": 28 }, "id": 9, "options": { @@ -1000,7 +1000,7 @@ "h": 7, "w": 10, "x": 10, - "y": 42 + "y": 28 }, "id": 1, "options": { @@ -1064,7 +1064,7 @@ "h": 7, "w": 5, "x": 0, - "y": 49 + "y": 35 }, "id": 8, "options": { @@ -1132,7 +1132,7 @@ "h": 7, "w": 5, "x": 5, - "y": 49 + "y": 35 }, "id": 7, "options": { @@ -1232,9 +1232,9 @@ "h": 7, "w": 10, "x": 10, - "y": 27 + "y": 35 }, - "id": 20, + "id": 21, "interval": "36", "options": { "legend": { @@ -1295,7 +1295,7 @@ "h": 7, "w": 5, "x": 0, - "y": 56 + "y": 42 }, "id": 2, "options": { @@ -1363,7 +1363,7 @@ "h": 7, "w": 5, "x": 5, - "y": 56 + "y": 42 }, "id": 5, "options": { @@ -1423,6 +1423,6 @@ "timezone": "browser", "title": "Aggregator Data", "uid": "aggregator", - "version": 6, + "version": 15, "weekStart": "" -} +} \ No newline at end of file From 8e10a7e7e73d175e2d3ba4489aba205522bb200f Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 10:01:09 -0300 Subject: [PATCH 037/135] chore: backoff dep --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 511ff4409..240f0dfce 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 - github.com/cenkalti/backoff/v4 v4.2.1 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/consensys/gnark v0.10.0 github.com/consensys/gnark-crypto v0.12.2-0.20240215234832-d72fcb379d3e github.com/fxamacker/cbor/v2 v2.7.0 diff --git a/go.sum b/go.sum index 7d13d545c..517063f42 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurT github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= From b1453b242fdf77397b50706941687fe310b7da36 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 11:47:55 -0300 Subject: [PATCH 038/135] chore: show only aggregator in bump metrics --- grafana/provisioning/dashboards/aligned/aggregator_batcher.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index 614dac92b..eb17d9f0b 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -1256,7 +1256,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "aligned_respond_to_task_gas_price_bumped", + "expr": "aligned_respond_to_task_gas_price_bumped{bot=\"aggregator\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, From 667095361e82193124c7501b921199b4fc7dbf0e Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 23 Oct 2024 12:25:37 -0300 Subject: [PATCH 039/135] separate retries into separate functions --- aggregator/internal/pkg/aggregator.go | 20 ++- aggregator/internal/pkg/server.go | 19 ++- core/chainio/avs_subscriber.go | 8 ++ core/chainio/avs_writer.go | 176 ++++++++++++++++---------- core/connection.go | 3 - core/connection_test.go | 81 ++++++++++-- core/utils/eth_client_utils.go | 8 +- 7 files changed, 222 insertions(+), 93 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index 9f514d090..ca724cbb8 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -270,7 +270,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.logger.Info("Sending aggregated response onchain", "taskIndex", blsAggServiceResp.TaskIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - //TODO: Retry() + _, err = agg.sendAggregatedResponse(batchIdentifierHash, batchData.BatchMerkleRoot, batchData.SenderAddress, nonSignerStakesAndSignature) if err == nil { agg.logger.Info("Aggregator successfully responded to task", @@ -288,7 +288,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.telemetry.LogTaskError(batchData.BatchMerkleRoot, err) } -// TODO: should we assume this is idempotent +// TODO(pat): should we be retrying the entire Send -> Receive Receipt step? For bumping the gas fee this is needed. // / Sends response to contract and waits for transaction receipt // / Returns error if it fails to send tx or receipt is not found @@ -300,6 +300,7 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "senderAddress", hex.EncodeToString(senderAddress[:]), "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) + // Retry txHash, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { agg.walletMutex.Unlock() @@ -311,6 +312,7 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Sending aggregated response for batch %s", hex.EncodeToString(batchIdentifierHash[:])) + // Retry receipt, err := utils.WaitForTransactionReceiptRetryable( agg.AggregatorConfig.BaseConfig.EthRpcClient, context.Background(), *txHash) if err != nil { @@ -365,10 +367,8 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(QUORUM_NUMBER)} quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(QUORUM_THRESHOLD)} - initilizeNewTask_func := func() error { - return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) - } - err := connection.Retry(initilizeNewTask_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + // Retry + err := agg.InitializeNewTaskRetryable(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) // FIXME(marian): When this errors, should we retry initializing new task? Logging fatal for now. if err != nil { agg.logger.Fatalf("BLS aggregation service error when initializing new task: %s", err) @@ -379,3 +379,11 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by agg.AggregatorConfig.BaseConfig.Logger.Info("- Unlocked Resources: Adding new task") agg.logger.Info("New task added", "batchIndex", batchIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) } + +func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { + initilizeNewTask_func := func() error { + // Returns an error if the + return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + } + return connection.Retry(initilizeNewTask_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} diff --git a/aggregator/internal/pkg/server.go b/aggregator/internal/pkg/server.go index b4f1867cf..dfd39f99b 100644 --- a/aggregator/internal/pkg/server.go +++ b/aggregator/internal/pkg/server.go @@ -7,6 +7,9 @@ import ( "net/rpc" "time" + "github.com/Layr-Labs/eigensdk-go/crypto/bls" + eigentypes "github.com/Layr-Labs/eigensdk-go/types" + connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/types" ) @@ -52,6 +55,7 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t taskIndex := uint32(0) ok := false + //TODO(pat): This is retried but waits are internal map aka is not a fallable connection for i := 0; i < waitForEventRetries; i++ { agg.taskMutex.Lock() agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Starting processing of Response") @@ -82,7 +86,8 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t agg.logger.Info("Starting bls signature process") go func() { - err := agg.blsAggregationService.ProcessNewSignature( + // Retry + err := agg.ProcessNewSignatureRetryable( context.Background(), taskIndex, signedTaskResponse.BatchIdentifierHash, &signedTaskResponse.BlsSignature, signedTaskResponse.OperatorId, ) @@ -121,3 +126,15 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { *reply = 1 return nil } + +func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { + processNewSignature_func := func() error { + return agg.blsAggregationService.ProcessNewSignature( + context.Background(), taskIndex, taskResponse, + blsSignature, operatorId, + ) + } + + return connection.Retry(processNewSignature_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + +} diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 16da4123f..ae9f9cab5 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -71,6 +71,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan // Subscribe to new tasks //TODO: Retry() + //TODO: Consolidate sub, err := subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) @@ -146,6 +147,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * // Subscribe to new tasks //TODO: Retry() + //TODO(pat): refactor fallbacks behind retry sub, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) @@ -153,6 +155,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * } //TODO: Retry() + //TODO(pat): refactor fallbacks behind retry subFallback, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) @@ -215,12 +218,17 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * return errorChannel, nil } +func watchNewBatchV2(opts *bind.WatchOpts, sink chan<- *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, batchMerkleRoot [][32]byte) (*event.Subscription, error) { + +} + func subscribeToNewTasksV2Retrayable( serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, logger sdklogging.Logger, ) (event.Subscription, error) { //TODO: Retry() + //TODO(pat): refactor fallbacks behind retry subscribe_func := func() (*event.Subscription, error) { sub, err := serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) return &sub, err diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 04fa85d52..ceb5389ce 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -74,19 +74,13 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*common.Hash, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction - respondToTaskV2_func := func() (*types.Transaction, error) { - return w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - } - tx, err := connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + // Retry + tx, err := w.respondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { - // Retry with fallback - tx, err = connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return nil, fmt.Errorf("transaction simulation failed: %v", err) - } + return nil, err } - err = w.checkRespondToTaskFeeLimitRetryable(tx, txOpts, batchIdentifierHash, senderAddress) + err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { return nil, err } @@ -94,16 +88,10 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // Send the transaction txOpts.NoSend = false txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit - respondToTaskV2_func = func() (*types.Transaction, error) { - return w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - } - tx, err = connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + // Retry + tx, err = w.respondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { - // Retry with fallback - tx, err = connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return nil, err - } + return nil, err } txHash := tx.Hash() @@ -111,32 +99,20 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return &txHash, nil } -func (w *AvsWriter) checkRespondToTaskFeeLimitRetryable(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { +func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { aggregatorAddress := txOpts.From simulatedCost := new(big.Int).Mul(new(big.Int).SetUint64(tx.Gas()), tx.GasPrice()) w.logger.Info("Simulated cost", "cost", simulatedCost) // Get RespondToTaskFeeLimit - //TODO: make this a public type - batchesState_func := func() (*struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - }, error) { - state, err := w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, batchIdentifierHash) - return &state, err - } - batchState, err := connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + // Retry + batchState, err := w.batchesStateRetryable(batchIdentifierHash) if err != nil { - // Retry with fallback - batchState, err = connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - // Fallback also failed - // Proceed to check values against simulated costs - w.logger.Error("Failed to get batch state", "error", err) - w.logger.Info("Proceeding with simulated cost checks") - return w.compareBalancesRetryable(simulatedCost, aggregatorAddress, senderAddress) - } + // Fallback also failed + // Proceed to check values against simulated costs + w.logger.Error("Failed to get batch state", "error", err) + w.logger.Info("Proceeding with simulated cost checks") + return w.compareBalances(simulatedCost, aggregatorAddress, senderAddress) } // At this point, batchState was successfully retrieved // Proceed to check values against RespondToTaskFeeLimit @@ -147,34 +123,33 @@ func (w *AvsWriter) checkRespondToTaskFeeLimitRetryable(tx *types.Transaction, t return fmt.Errorf("cost of transaction is higher than Batch.RespondToTaskFeeLimit") } - return w.compareBalancesRetryable(respondToTaskFeeLimit, aggregatorAddress, senderAddress) + // Retry + return w.compareBalances(respondToTaskFeeLimit, aggregatorAddress, senderAddress) } -func (w *AvsWriter) compareBalancesRetryable(amount *big.Int, aggregatorAddress common.Address, senderAddress [20]byte) error { - if err := w.compareAggregatorBalanceRetryable(amount, aggregatorAddress); err != nil { +func (w *AvsWriter) compareBalances(amount *big.Int, aggregatorAddress common.Address, senderAddress [20]byte) error { + // Retry + if err := w.compareAggregatorBalance(amount, aggregatorAddress); err != nil { return err } - if err := w.compareBatcherBalanceRetryable(amount, senderAddress); err != nil { + // Retry + if err := w.compareBatcherBalance(amount, senderAddress); err != nil { return err } return nil } -func (w *AvsWriter) compareAggregatorBalanceRetryable(amount *big.Int, aggregatorAddress common.Address) error { +func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress common.Address) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - // Get Agg wallet balance - balanceAt_func := func() (*big.Int, error) { - return w.Client.BalanceAt(ctx, aggregatorAddress, nil) - } - aggregatorBalance, err := connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + + // TODO(pat): Separate Retry from call + // Retry + aggregatorBalance, err := w.balanceAtRetryable(ctx, aggregatorAddress, nil) if err != nil { - aggregatorBalance, err = connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - // Ignore and continue. - w.logger.Error("failed to get aggregator balance: %v", err) - return nil - } + // Ignore and continue. + w.logger.Error("failed to get aggregator balance: %v", err) + return nil } w.logger.Info("Aggregator balance", "balance", aggregatorBalance) if aggregatorBalance.Cmp(amount) < 0 { @@ -183,19 +158,15 @@ func (w *AvsWriter) compareAggregatorBalanceRetryable(amount *big.Int, aggregato return nil } -func (w *AvsWriter) compareBatcherBalanceRetryable(amount *big.Int, senderAddress [20]byte) error { +func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byte) error { // Get batcher balance - batcherBalances_func := func() (*big.Int, error) { - return w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) - } - batcherBalance, err := connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + // TODO(pat): Separate Retry from call + // Retry + batcherBalance, err := w.batcherBalancesRetryable(senderAddress) if err != nil { - batcherBalance, err = connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - // Ignore and continue. - w.logger.Error("Failed to get batcherBalance", "error", err) - return nil - } + // Ignore and continue. + w.logger.Error("Failed to get batcherBalance", "error", err) + return nil } w.logger.Info("Batcher balance", "balance", batcherBalance) if batcherBalance.Cmp(amount) < 0 { @@ -203,3 +174,76 @@ func (w *AvsWriter) compareBatcherBalanceRetryable(amount *big.Int, senderAddres } return nil } + +func (w *AvsWriter) respondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { + respondToTaskV2_func := func() (*types.Transaction, error) { + tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + // Retry with fallback + tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + return nil, err + } + } + return tx, err + } + return connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (w *AvsWriter) batchesStateRetryable(arg0 [32]byte) (struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int +}, error) { + batchesState_func := func() (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err := w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) + if err != nil { + // Retry with fallback + state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) + if err != nil { + return nil, err + } + } + return &state, err + } + state, err := connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return *state, err +} + +// TODO: separate retry concerns and inline them in real functions +func (w *AvsWriter) batcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { + batcherBalances_func := func() (*big.Int, error) { + batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + if err != nil { + batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) + if err != nil { + return nil, err + } + } + return batcherBalance, err + } + return connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +// TODO: separate retry concerns and inline them in real functions +func (w *AvsWriter) balanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { + + balanceAt_func := func() (*big.Int, error) { + aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) + //aggregatorBalance, err := connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + if err != nil { + aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) + //aggregatorBalance, err = connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + if err != nil { + // Ignore and continue. + return nil, err + } + } + return aggregatorBalance, err + } + return connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} diff --git a/core/connection.go b/core/connection.go index f023d382f..6c84e49d7 100644 --- a/core/connection.go +++ b/core/connection.go @@ -23,9 +23,6 @@ const MinDelay = 1000 const RetryFactor = 2 const NumRetries = 3 -//NOTE: I reverted the change Marcos made to make the return type a *T as some return types of the function calls. -// were not pointers or uint64. - // Same as Retry only that the functionToRetry can return a value upon correct execution func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { i := 0 diff --git a/core/connection_test.go b/core/connection_test.go index 18e58b59f..c27e22309 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -19,7 +19,9 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" + servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" connection "github.com/yetanotherco/aligned_layer/core" + "github.com/yetanotherco/aligned_layer/core/chainio" "github.com/yetanotherco/aligned_layer/core/utils" ) @@ -131,22 +133,27 @@ func TestAnvilSetupKill(t *testing.T) { assert.Nil(t, err, "Anvil Process Killed") } +// |--Aggreagator Retry Tests--| + // Waits for receipt from anvil node -> Will fail to get receipt func TestWaitForTransactionReceiptRetryable(t *testing.T) { // Retry call Params - to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + to := common.BytesToAddress([]byte{0x11}) tx := types.NewTx(&types.AccessListTx{ + ChainID: big.NewInt(1337), Nonce: 1, - GasPrice: big.NewInt(500), - Gas: 1000000, + GasPrice: big.NewInt(11111), + Gas: 1111, To: &to, - Value: big.NewInt(1), + Value: big.NewInt(111), + Data: []byte{0x11, 0x11, 0x11}, }) ctx := context.WithoutCancel(context.Background()) hash := tx.Hash() + // Start anvil cmd, client, err := SetupAnvil(8545) if err != nil { @@ -155,7 +162,11 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { // Assert Call succeeds why Anvil running _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - assert.NotNil(t, err, "Call to Anvil failed") + assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) + if err.Error() != "not found" { + fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) + return + } // Kill Anvil if err := cmd.Process.Kill(); err != nil { @@ -163,29 +174,75 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } + // TODO: wait for transaction returns nil for receipt and err if nno connection. // Check that permanent error thrown - _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - if err != nil { - t.Errorf("Retry error!: %s", err) - } + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + assert.Nil(t, receipt, "Receipt not empty") + assert.NotNil(t, err, "Retry ") + fmt.Printf("emitted error: %s\n", err) // Start anvil _, client, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } + _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) assert.NotNil(t, err, "Call to Anvil failed") + if err.Error() != "not found" { + fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) + } +} + +func TestSendAggregatedResponseRetryable(t *testing.T) { +} + +func TestInitializeNewTaskRetryable(t *testing.T) { + //TODO: Instantiate Aggregator +} + +// |--Subscriber Retry Tests--| + +func TestSubscribeToNewTasksV3Retryable(t *testing.T) { + newBatchChan := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + + baseConfig := core.NewBaseConfig("") + avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(baseConfig) + if err != nil { + return nil, err + } + + agg.taskSubscriber, err = avsSubscriber.SubscribeToNewTasksV3Retryable(newBatchChan) } +// |--Server Retry Tests--| +func TestProcessNewSignatureRetryable(t *testing.T) { +} + +// |--AVS-Writer Retry Tests--| + func TestRespondToTaskV2(t *testing.T) { } -func TestRespondToTaskV3(t *testing.T) { +func TestBatchesState(t *testing.T) { +} + +func TestBalanceAt(t *testing.T) { +} + +func TestBatchersBalances(t *testing.T) { +} + +// |--AVS-Subscriber Retry Tests--| + +func TestSubscribeToNewTasksV2(t *testing.T) { +} + +func TestWatchNewBatchV2(t *testing.T) { } -func SubscribeToNewTasksV2(t *testing.T) { +func TestSubscribeToNewTasksV3(t *testing.T) { } -func SubscribeToNewTasksV3(t *testing.T) { +func TestWatchNewBatchV3(t *testing.T) { } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 4cef9d7b5..e6291487d 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -10,13 +10,11 @@ import ( connection "github.com/yetanotherco/aligned_layer/core" ) +// Does not func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { + // For if no receipt and no error TransactionReceipt return "not found" as an error catch all ref: https://github.com/ethereum/go-ethereum/blob/master/ethclient/ethclient.go#L313 receipt_func := func() (*types.Receipt, error) { return client.TransactionReceipt(ctx, txHash) } - receipt, err := connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return nil, err - } - return receipt, nil + return connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { From 776dc25ae6c718684d41da8cebb1203e0d0571b9 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 23 Oct 2024 14:04:24 -0300 Subject: [PATCH 040/135] abstract retries in avs subscriber --- core/chainio/avs_subscriber.go | 179 +++++++++++++++++---------------- 1 file changed, 91 insertions(+), 88 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index ae9f9cab5..0224a5d2b 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -146,16 +146,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) // Subscribe to new tasks - //TODO: Retry() - //TODO(pat): refactor fallbacks behind retry + // Retry() sub, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err } - //TODO: Retry() - //TODO(pat): refactor fallbacks behind retry + // Retry() subFallback, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) @@ -198,7 +196,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - //TODO: Retry() + // Retry() sub, err = subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -206,7 +204,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - //TODO: Retry() + // Retry() subFallback, err = subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -218,25 +216,17 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * return errorChannel, nil } -func watchNewBatchV2(opts *bind.WatchOpts, sink chan<- *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, batchMerkleRoot [][32]byte) (*event.Subscription, error) { - -} - func subscribeToNewTasksV2Retrayable( serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, logger sdklogging.Logger, ) (event.Subscription, error) { - //TODO: Retry() - //TODO(pat): refactor fallbacks behind retry subscribe_func := func() (*event.Subscription, error) { sub, err := serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) return &sub, err } sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - // If the retries stop early we propogate the Permanent error to the caller. - // TODO: ask if this is the behavior we want. return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) } logger.Info("Subscribed to new AlignedLayer tasks") @@ -248,7 +238,6 @@ func subscribeToNewTasksV3Retryable( newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, logger sdklogging.Logger, ) (event.Subscription, error) { - //TODO: Retry() subscribe_func := func() (*event.Subscription, error) { sub, err := serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) return &sub, err @@ -314,21 +303,75 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL } } -// getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. -func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { - - //TODO: Retry() +func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (*uint64, error) { latestBlock_func := func() (*uint64, error) { - sub, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) - return &sub, err + latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) + if err != nil { + latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) + if err != nil { + return nil, err + } + } + return &latestBlock, err } - latestBlock, err := connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - //TODO: Retry() - latestBlock, err = connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) filterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + } + return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) filterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { + + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { + return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + } + return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) batchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int +}, error) { + batchState_func := func() (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) + return &state, err + } + + return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) subscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (*ethereum.Subscription, error) { + subscribeNewHead_func := func() (*ethereum.Subscription, error) { + sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) if err != nil { - return nil, err + //TODO: Retry() + sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) + if err != nil { + return nil, err + } } + return &sub, err + } + return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +// getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. +func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { + + // Retry() + latestBlock, err := s.BlockNumberRetryable(context.Background()) + if err != nil { + return nil, err } var fromBlock uint64 @@ -340,11 +383,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*ser } //TODO: Retry() - filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) - } - //logs, err := s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) - logs, err := connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + logs, err := s.filterBatchV2Retryable(fromBlock, context.Background()) if err != nil { return nil, err } @@ -367,16 +406,19 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*ser batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) //TODO: Retry() - batchState_func := func() (*struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - }, error) { - state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) - return &state, err - } + /* + batchState_func := func() (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) + return &state, err + } - state, err := connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + state, err := connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + */ + state, err := s.batchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -390,20 +432,10 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*ser // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { - //TODO: Retry() - latestBlock_func := func() (*uint64, error) { - latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) - return &latestBlock, err - } - //latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) - latestBlock, err := connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + // Retry() + latestBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { - //TODO: Retry() - //s.AvsContractBindings.ethClientFallback.BlockNumber(context.Background()) - latestBlock, err = connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return nil, err - } + return nil, err } var fromBlock uint64 @@ -415,10 +447,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*ser } //TODO: Retry() - filterNewBatchV3_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) - } - logs, err := connection.RetryWithData(filterNewBatchV3_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + logs, err := s.filterBatchV3Retryable(fromBlock, context.Background()) if err != nil { return nil, err } @@ -441,16 +470,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*ser batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) //TODO: Retry() - batchesState_func := func() (*struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - }, error) { - state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) - return &state, err - } - //state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) - state, err := connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + state, err := s.batchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -464,35 +484,18 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*ser func (s *AvsSubscriber) WaitForOneBlockRetryable(startBlock uint64) error { //TODO: Retry() - blockNum_func := func() (*uint64, error) { - block_num, err := s.AvsContractBindings.ethClient.BlockNumber(context.Background()) - return &block_num, err - } - currentBlock, err := connection.RetryWithData(blockNum_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + currentBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { - // try with the fallback client - //TODO: Retry() - currentBlock, err = connection.RetryWithData(blockNum_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return err - } + return err } if *currentBlock <= startBlock { // should really be == but just in case // Subscribe to new head c := make(chan *types.Header) //TODO: Retry() - subscribeNewHead_func := func() (*ethereum.Subscription, error) { - sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(context.Background(), c) - return &sub, err - } - sub, err := connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + sub, err := s.subscribeNewHeadRetryable(context.Background(), c) if err != nil { - //TODO: Retry() - sub, err = connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return err - } + return err } // Read channel for the new block From 959694b622f86e4fe85982f0b07c32f12a709928 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 14:20:27 -0300 Subject: [PATCH 041/135] refactor: wait for transaction receipt check if receipt exist before returning ctx timeout err --- core/utils/eth_client_utils.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 08dd4c911..ec0abb8ac 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -18,15 +18,15 @@ const sleepTime = 5 * time.Second func WaitForTransactionReceipt(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { for i := 0; i < maxRetries; i++ { - receipt, err := client.TransactionReceipt(ctx, txHash) + receipt, _ := client.TransactionReceipt(ctx, txHash) + if receipt != nil { + return receipt, nil + } // if context has timed out, return if ctx.Err() != nil { return nil, ctx.Err() - } - if err != nil { - time.Sleep(sleepTime) } else { - return receipt, nil + time.Sleep(sleepTime) } } return nil, fmt.Errorf("transaction receipt not found for txHash: %s", txHash.String()) From c875e4bead53a49cb9920f16ff119fcc8a254586 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 23 Oct 2024 15:42:24 -0300 Subject: [PATCH 042/135] save --- aggregator/internal/pkg/aggregator.go | 2 +- core/chainio/avs_subscriber.go | 39 ++++++------------ core/connection_test.go | 58 +++++++++++++++------------ 3 files changed, 46 insertions(+), 53 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index ca724cbb8..a712080df 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -263,7 +263,7 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:]), "taskCreatedBlock", taskCreatedBlock) - err := agg.avsSubscriber.WaitForOneBlockRetryable(taskCreatedBlock) + err := agg.avsSubscriber.WaitForOneBlock(taskCreatedBlock) if err != nil { agg.logger.Error("Error waiting for one block, sending anyway", "err", err) } diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 0224a5d2b..67e12e702 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -101,7 +101,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan s.processNewBatchV2(newBatch, batchesSet, newBatchMutex, newTaskCreatedChan) case <-pollLatestBatchTicker.C: //TODO: Retry() - latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV2Retryable() + latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV2() if err != nil { s.logger.Debug("Failed to get latest task from blockchain", "err", err) continue @@ -303,7 +303,7 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL } } -func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (*uint64, error) { +func (s *AvsSubscriber) blockNumberRetryable(ctx context.Context) (*uint64, error) { latestBlock_func := func() (*uint64, error) { latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) if err != nil { @@ -354,7 +354,6 @@ func (s *AvsSubscriber) subscribeNewHeadRetryable(ctx context.Context, c chan<- subscribeNewHead_func := func() (*ethereum.Subscription, error) { sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) if err != nil { - //TODO: Retry() sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) if err != nil { return nil, err @@ -366,10 +365,10 @@ func (s *AvsSubscriber) subscribeNewHeadRetryable(ctx context.Context, c chan<- } // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. -func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { +func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { // Retry() - latestBlock, err := s.BlockNumberRetryable(context.Background()) + latestBlock, err := s.blockNumberRetryable(context.Background()) if err != nil { return nil, err } @@ -382,7 +381,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*ser fromBlock = *latestBlock - BlockInterval } - //TODO: Retry() + // Retry() logs, err := s.filterBatchV2Retryable(fromBlock, context.Background()) if err != nil { return nil, err @@ -405,19 +404,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*ser batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - //TODO: Retry() - /* - batchState_func := func() (*struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - }, error) { - state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(nil, batchIdentifierHash) - return &state, err - } - - state, err := connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - */ + // Retry() state, err := s.batchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err @@ -433,7 +420,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2Retryable() (*ser // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { // Retry() - latestBlock, err := s.BlockNumberRetryable(context.Background()) + latestBlock, err := s.blockNumberRetryable(context.Background()) if err != nil { return nil, err } @@ -446,7 +433,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*ser fromBlock = *latestBlock - BlockInterval } - //TODO: Retry() + // Retry() logs, err := s.filterBatchV3Retryable(fromBlock, context.Background()) if err != nil { return nil, err @@ -469,7 +456,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*ser batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - //TODO: Retry() + // Retry() state, err := s.batchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err @@ -482,9 +469,9 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*ser return lastLog, nil } -func (s *AvsSubscriber) WaitForOneBlockRetryable(startBlock uint64) error { - //TODO: Retry() - currentBlock, err := s.BlockNumberRetryable(context.Background()) +func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { + // Retry() + currentBlock, err := s.blockNumberRetryable(context.Background()) if err != nil { return err } @@ -492,7 +479,7 @@ func (s *AvsSubscriber) WaitForOneBlockRetryable(startBlock uint64) error { if *currentBlock <= startBlock { // should really be == but just in case // Subscribe to new head c := make(chan *types.Header) - //TODO: Retry() + // Retry() sub, err := s.subscribeNewHeadRetryable(context.Background(), c) if err != nil { return err diff --git a/core/connection_test.go b/core/connection_test.go index c27e22309..73adf89b5 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -1,7 +1,6 @@ package connection_test import ( - "bytes" "context" "fmt" "log" @@ -19,9 +18,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" connection "github.com/yetanotherco/aligned_layer/core" - "github.com/yetanotherco/aligned_layer/core/chainio" "github.com/yetanotherco/aligned_layer/core/utils" ) @@ -75,10 +72,7 @@ func SetupAnvil(port uint16) (*exec.Cmd, *eth.InstrumentedClient, error) { // Create a command cmd := exec.Command("anvil", "--port", port_str, "--load-state", "../contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json", "--block-time", "7") - - // Prepare to capture the output - var out bytes.Buffer - cmd.Stdout = &out + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} // Run the command err = cmd.Start() @@ -174,12 +168,10 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } - // TODO: wait for transaction returns nil for receipt and err if nno connection. - // Check that permanent error thrown + // Fails receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) assert.Nil(t, receipt, "Receipt not empty") - assert.NotNil(t, err, "Retry ") - fmt.Printf("emitted error: %s\n", err) + assert.NotEqual(t, err.Error(), "not found") // Start anvil _, client, err = SetupAnvil(8545) @@ -194,6 +186,8 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { } } +/* + func TestSendAggregatedResponseRetryable(t *testing.T) { } @@ -201,22 +195,24 @@ func TestInitializeNewTaskRetryable(t *testing.T) { //TODO: Instantiate Aggregator } +// |--Server Retry Tests--| +func TestProcessNewSignatureRetryable(t *testing.T) { + agg := NewAggregator() + agg.ProcessNewSignatureRetryable() +} + // |--Subscriber Retry Tests--| func TestSubscribeToNewTasksV3Retryable(t *testing.T) { - newBatchChan := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + newBatchChan := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) - baseConfig := core.NewBaseConfig("") - avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(baseConfig) - if err != nil { - return nil, err - } - - agg.taskSubscriber, err = avsSubscriber.SubscribeToNewTasksV3Retryable(newBatchChan) -} + baseConfig := core.NewBaseConfig("") + avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(baseConfig) + if err != nil { + return nil, err + } -// |--Server Retry Tests--| -func TestProcessNewSignatureRetryable(t *testing.T) { + agg.taskSubscriber, err = avsSubscriber.SubscribeToNewTasksV3Retryable(newBatchChan) } // |--AVS-Writer Retry Tests--| @@ -224,7 +220,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { func TestRespondToTaskV2(t *testing.T) { } -func TestBatchesState(t *testing.T) { +func TestBatchesStateWriter(t *testing.T) { } func TestBalanceAt(t *testing.T) { @@ -238,11 +234,21 @@ func TestBatchersBalances(t *testing.T) { func TestSubscribeToNewTasksV2(t *testing.T) { } -func TestWatchNewBatchV2(t *testing.T) { +func TestSubscribeToNewTasksV3(t *testing.T) { +} + +func TestBlockNumber(t *testing.T) { } -func TestSubscribeToNewTasksV3(t *testing.T) { +func TestFilterBatchV2(t *testing.T) { } -func TestWatchNewBatchV3(t *testing.T) { +func TestFilterBatchV3(t *testing.T) { } + +func TestBatchesStateSubscriber(t *testing.T) { +} + +func TestSubscribeNewHead(t *testing.T) { +} +*/ From 9eef24f20394299070eb32dc51c962cce6053f92 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 17:13:04 -0300 Subject: [PATCH 043/135] reafactor: receipt timeout constant --- core/chainio/avs_writer.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 2cb068aaf..9f91b5f9b 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -21,7 +21,9 @@ import ( ) const ( - gasBumpPercentage int = 20 + GasBumpPercentage int = 20 + // wait as much as 3 blocks time for the receipt + SendAggregateResponseReceiptTimeout time.Duration = time.Second * 36 ) type AvsWriter struct { @@ -119,7 +121,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, fmt.Errorf("transaction simulation failed: %v", err) } } - txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, gasBumpPercentage, i) + txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, GasBumpPercentage, i) w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) @@ -137,7 +139,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe } } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*36) + ctx, cancel := context.WithTimeout(context.Background(), SendAggregateResponseReceiptTimeout) defer cancel() receipt, err := utils.WaitForTransactionReceipt(w.Client, ctx, tx.Hash()) From 053d1fb8b824e5822c974751b813b1aeceb82e42 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 17:54:21 -0300 Subject: [PATCH 044/135] test: connection retries --- core/connection_test.go | 67 ++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/core/connection_test.go b/core/connection_test.go index a922ec10e..acedf0eeb 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -17,26 +17,65 @@ func DummyFunction(x uint64) (uint64, error) { return x, nil } +// here we run the dummy function and with the retry and check: +// - The number of retries checks based on the `n` +// - The returned valued matches based on the `n` +// - The returned err matches based on the `n` func TestRetryWithData(t *testing.T) { - function := func() (*uint64, error) { - x, err := DummyFunction(43) - return &x, err + retries := -1 + testFun := func(n uint64) func() (*uint64, error) { + return func() (*uint64, error) { + retries++ + x, err := DummyFunction(n) + return &x, err + } } - data, err := connection.RetryWithData(function, 1000, 2, 3) - if err != nil { - t.Errorf("Retry error!: %s", err) - } else { - fmt.Printf("DATA: %d\n", data) + data, err := connection.RetryWithData(testFun(uint64(41)), 1000, 2, 3) + if !(retries == 3 && *data == 0 && err != nil) { + t.Error("Incorrect execution when n == 41") + } + //restart + retries = -1 + data, err = connection.RetryWithData(testFun(42), 1000, 2, 3) + if !(retries == 0 && data == nil) { + if _, ok := err.(*connection.PermanentError); ok { + t.Error("Incorrect execution when n == 42") + } + } + //restart + retries = -1 + data, err = connection.RetryWithData(testFun(43), 1000, 2, 3) + if !(retries == 0 && *data == 43 && err == nil) { + t.Error("Incorrect execution when n == 43") } } +// same as above but without checking returned data func TestRetry(t *testing.T) { - function := func() error { - _, err := DummyFunction(43) - return err + retries := -1 + testFun := func(n uint64) func() error { + return func() error { + retries++ + _, err := DummyFunction(n) + return err + } + } + err := connection.Retry(testFun(uint64(41)), 1000, 2, 3) + if !(retries == 3 && err != nil) { + t.Error("Incorrect execution when n == 41") + } + //restart + retries = -1 + err = connection.Retry(testFun(42), 1000, 2, 3) + if !(retries == 0) { + if _, ok := err.(*connection.PermanentError); ok { + t.Error("Incorrect execution when n == 42") + } } - err := connection.Retry(function, 1000, 2, 3) - if err != nil { - t.Errorf("Retry error!: %s", err) + //restart + retries = -1 + err = connection.Retry(testFun(43), 1000, 2, 3) + if !(retries == 0 && err == nil) { + t.Error("Incorrect execution when n == 43") } } From 24a96ffbe3eb57a7ab5b92baaf1aaf0e4273e01e Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 23 Oct 2024 18:08:30 -0300 Subject: [PATCH 045/135] docs: typo in retry test [skip ci] --- core/connection_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/connection_test.go b/core/connection_test.go index acedf0eeb..2a0918f5c 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -17,7 +17,7 @@ func DummyFunction(x uint64) (uint64, error) { return x, nil } -// here we run the dummy function and with the retry and check: +// here we run the dummy function with the retry and check: // - The number of retries checks based on the `n` // - The returned valued matches based on the `n` // - The returned err matches based on the `n` From eecf2a4f0e6f65f0bcfb5ba28fbbc6055264dfa5 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 23 Oct 2024 21:44:58 -0300 Subject: [PATCH 046/135] reorg --- aggregator/internal/pkg/aggregator.go | 2 + aggregator/internal/pkg/server.go | 3 +- core/chainio/avs_subscriber.go | 207 +++++++++++++------------- core/chainio/avs_writer.go | 3 +- core/connection_test.go | 20 +-- 5 files changed, 120 insertions(+), 115 deletions(-) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/internal/pkg/aggregator.go index a712080df..9907c3251 100644 --- a/aggregator/internal/pkg/aggregator.go +++ b/aggregator/internal/pkg/aggregator.go @@ -380,6 +380,8 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by agg.logger.Info("New task added", "batchIndex", batchIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) } +// |---RETRYABLE---| + func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { initilizeNewTask_func := func() error { // Returns an error if the diff --git a/aggregator/internal/pkg/server.go b/aggregator/internal/pkg/server.go index dfd39f99b..013ff4bd4 100644 --- a/aggregator/internal/pkg/server.go +++ b/aggregator/internal/pkg/server.go @@ -127,6 +127,8 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { return nil } +// |---RETRYABLE---| + func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { processNewSignature_func := func() error { return agg.blsAggregationService.ProcessNewSignature( @@ -136,5 +138,4 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd } return connection.Retry(processNewSignature_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - } diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 67e12e702..47c94fd1f 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -70,15 +70,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) // Subscribe to new tasks - //TODO: Retry() - //TODO: Consolidate + // Retry() sub, err := subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err } - //TODO: Retry() + // Retry() subFallback, err := subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) @@ -121,7 +120,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - //TODO: Retry() + // Retry() sub, err = subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -129,7 +128,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - //TODO: Retry() + // Retry() subFallback, err = subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -176,7 +175,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * s.processNewBatchV3(newBatch, batchesSet, newBatchMutex, newTaskCreatedChan) case <-pollLatestBatchTicker.C: //TODO: Retry() - latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV3Retryable() + latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV3() if err != nil { s.logger.Debug("Failed to get latest task from blockchain", "err", err) continue @@ -216,41 +215,6 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * return errorChannel, nil } -func subscribeToNewTasksV2Retrayable( - serviceManager *servicemanager.ContractAlignedLayerServiceManager, - newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, - logger sdklogging.Logger, -) (event.Subscription, error) { - subscribe_func := func() (*event.Subscription, error) { - sub, err := serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) - return &sub, err - } - sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) - } - logger.Info("Subscribed to new AlignedLayer tasks") - return *sub, nil -} - -func subscribeToNewTasksV3Retryable( - serviceManager *servicemanager.ContractAlignedLayerServiceManager, - newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, - logger sdklogging.Logger, -) (event.Subscription, error) { - subscribe_func := func() (*event.Subscription, error) { - sub, err := serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) - return &sub, err - } - sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - // If the retries stop early we propogate the Permanent error to the caller. - return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) - } - logger.Info("Subscribed to new AlignedLayer tasks") - return *sub, nil -} - func (s *AvsSubscriber) processNewBatchV2(batch *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, batchesSet map[[32]byte]struct{}, newBatchMutex *sync.Mutex, newTaskCreatedChan chan<- *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) { newBatchMutex.Lock() defer newBatchMutex.Unlock() @@ -303,67 +267,6 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL } } -func (s *AvsSubscriber) blockNumberRetryable(ctx context.Context) (*uint64, error) { - latestBlock_func := func() (*uint64, error) { - latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) - if err != nil { - latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) - if err != nil { - return nil, err - } - } - return &latestBlock, err - } - return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) -} - -func (s *AvsSubscriber) filterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - - filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) - } - return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) -} - -func (s *AvsSubscriber) filterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - - filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) - } - return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) -} - -func (s *AvsSubscriber) batchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (*struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int -}, error) { - batchState_func := func() (*struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - }, error) { - state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) - return &state, err - } - - return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) -} - -func (s *AvsSubscriber) subscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (*ethereum.Subscription, error) { - subscribeNewHead_func := func() (*ethereum.Subscription, error) { - sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) - if err != nil { - sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) - if err != nil { - return nil, err - } - } - return &sub, err - } - return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) -} - // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { @@ -418,7 +321,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag } // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. -func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3Retryable() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { +func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { // Retry() latestBlock, err := s.blockNumberRetryable(context.Background()) if err != nil { @@ -507,3 +410,101 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // func (s *AvsSubscriber) ParseTaskResponded(rawLog types.Log) (*cstaskmanager.ContractAlignedLayerTaskManagerTaskResponded, error) { // return s.AvsContractBindings.TaskManager.ContractAlignedLayerTaskManagerFilterer.ParseTaskResponded(rawLog) // } + +// |---RETRYABLE---| + +func (s *AvsSubscriber) blockNumberRetryable(ctx context.Context) (*uint64, error) { + latestBlock_func := func() (*uint64, error) { + latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) + if err != nil { + latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) + if err != nil { + return nil, err + } + } + return &latestBlock, err + } + return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) filterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + } + return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) filterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { + + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { + return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + } + return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) batchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int +}, error) { + batchState_func := func() (*struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) + return &state, err + } + + return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func (s *AvsSubscriber) subscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (*ethereum.Subscription, error) { + subscribeNewHead_func := func() (*ethereum.Subscription, error) { + sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) + if err != nil { + sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) + if err != nil { + return nil, err + } + } + return &sub, err + } + return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) +} + +func subscribeToNewTasksV2Retrayable( + serviceManager *servicemanager.ContractAlignedLayerServiceManager, + newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, + logger sdklogging.Logger, +) (event.Subscription, error) { + subscribe_func := func() (*event.Subscription, error) { + sub, err := serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) + return &sub, err + } + sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + if err != nil { + return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + } + logger.Info("Subscribed to new AlignedLayer tasks") + return *sub, nil +} + +func subscribeToNewTasksV3Retryable( + serviceManager *servicemanager.ContractAlignedLayerServiceManager, + newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, + logger sdklogging.Logger, +) (event.Subscription, error) { + subscribe_func := func() (*event.Subscription, error) { + sub, err := serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) + return &sub, err + } + sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + if err != nil { + // If the retries stop early we propogate the Permanent error to the caller. + return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + } + logger.Info("Subscribed to new AlignedLayer tasks") + return *sub, nil +} diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index ceb5389ce..b8404316b 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -175,6 +175,8 @@ func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byt return nil } +// |---RETRYABLE---| + func (w *AvsWriter) respondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { respondToTaskV2_func := func() (*types.Transaction, error) { tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) @@ -229,7 +231,6 @@ func (w *AvsWriter) batcherBalancesRetryable(senderAddress common.Address) (*big return connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -// TODO: separate retry concerns and inline them in real functions func (w *AvsWriter) balanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { balanceAt_func := func() (*big.Int, error) { diff --git a/core/connection_test.go b/core/connection_test.go index 73adf89b5..3cbf38114 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -168,7 +168,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } - // Fails + // Fails with "not found" receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) assert.Nil(t, receipt, "Receipt not empty") assert.NotEqual(t, err.Error(), "not found") @@ -197,22 +197,22 @@ func TestInitializeNewTaskRetryable(t *testing.T) { // |--Server Retry Tests--| func TestProcessNewSignatureRetryable(t *testing.T) { - agg := NewAggregator() - agg.ProcessNewSignatureRetryable() + agg := NewAggregator() + agg.ProcessNewSignatureRetryable() } // |--Subscriber Retry Tests--| func TestSubscribeToNewTasksV3Retryable(t *testing.T) { - newBatchChan := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + newBatchChan := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) - baseConfig := core.NewBaseConfig("") - avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(baseConfig) - if err != nil { - return nil, err - } + baseConfig := core.NewBaseConfig("") + avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(baseConfig) + if err != nil { + return nil, err + } - agg.taskSubscriber, err = avsSubscriber.SubscribeToNewTasksV3Retryable(newBatchChan) + agg.taskSubscriber, err = avsSubscriber.SubscribeToNewTasksV3Retryable(newBatchChan) } // |--AVS-Writer Retry Tests--| From d51a16cb1cfeeba67d90bf427d819b7043c2d459 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 24 Oct 2024 13:01:11 -0300 Subject: [PATCH 047/135] export aggregator package --- aggregator/{internal => }/pkg/aggregator.go | 0 aggregator/{internal => }/pkg/server.go | 3 ++- aggregator/{internal => }/pkg/subscriber.go | 2 +- aggregator/{internal => }/pkg/telemetry.go | 0 4 files changed, 3 insertions(+), 2 deletions(-) rename aggregator/{internal => }/pkg/aggregator.go (100%) rename aggregator/{internal => }/pkg/server.go (98%) rename aggregator/{internal => }/pkg/subscriber.go (96%) rename aggregator/{internal => }/pkg/telemetry.go (100%) diff --git a/aggregator/internal/pkg/aggregator.go b/aggregator/pkg/aggregator.go similarity index 100% rename from aggregator/internal/pkg/aggregator.go rename to aggregator/pkg/aggregator.go diff --git a/aggregator/internal/pkg/server.go b/aggregator/pkg/server.go similarity index 98% rename from aggregator/internal/pkg/server.go rename to aggregator/pkg/server.go index 013ff4bd4..51ae435d9 100644 --- a/aggregator/internal/pkg/server.go +++ b/aggregator/pkg/server.go @@ -129,10 +129,11 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // |---RETRYABLE---| +// Error throw is ______ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { processNewSignature_func := func() error { return agg.blsAggregationService.ProcessNewSignature( - context.Background(), taskIndex, taskResponse, + ctx, taskIndex, taskResponse, blsSignature, operatorId, ) } diff --git a/aggregator/internal/pkg/subscriber.go b/aggregator/pkg/subscriber.go similarity index 96% rename from aggregator/internal/pkg/subscriber.go rename to aggregator/pkg/subscriber.go index b592f6ace..4aef715da 100644 --- a/aggregator/internal/pkg/subscriber.go +++ b/aggregator/pkg/subscriber.go @@ -24,7 +24,7 @@ func (agg *Aggregator) SubscribeToNewTasks() error { func (agg *Aggregator) subscribeToNewTasks() error { var err error - agg.taskSubscriber, err = agg.avsSubscriber.SubscribeToNewTasksV3Retryable(agg.NewBatchChan) + agg.taskSubscriber, err = agg.avsSubscriber.SubscribeToNewTasksV3(agg.NewBatchChan) if err != nil { agg.AggregatorConfig.BaseConfig.Logger.Info("Failed to create task subscriber", "err", err) diff --git a/aggregator/internal/pkg/telemetry.go b/aggregator/pkg/telemetry.go similarity index 100% rename from aggregator/internal/pkg/telemetry.go rename to aggregator/pkg/telemetry.go From e1fa684850ea66d23e20f78a1651ebfebec930ca Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 24 Oct 2024 13:01:26 -0300 Subject: [PATCH 048/135] modify main --- aggregator/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/cmd/main.go b/aggregator/cmd/main.go index 9a4cfed7d..399ebeac7 100644 --- a/aggregator/cmd/main.go +++ b/aggregator/cmd/main.go @@ -7,7 +7,7 @@ import ( "os" "github.com/urfave/cli/v2" - "github.com/yetanotherco/aligned_layer/aggregator/internal/pkg" + "github.com/yetanotherco/aligned_layer/aggregator/pkg" "github.com/yetanotherco/aligned_layer/core/config" ) From 25b93b1f24c76af2b1559fbacfe419eaf3a8dd33 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 24 Oct 2024 13:01:35 -0300 Subject: [PATCH 049/135] add config file for testing --- config-files/config-aggregator-test.yaml | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 config-files/config-aggregator-test.yaml diff --git a/config-files/config-aggregator-test.yaml b/config-files/config-aggregator-test.yaml new file mode 100644 index 000000000..31f934f41 --- /dev/null +++ b/config-files/config-aggregator-test.yaml @@ -0,0 +1,54 @@ +# Common variables for all the services +# 'production' only prints info and above. 'development' also prints debug +environment: "production" +aligned_layer_deployment_config_file_path: "../contracts/script/output/devnet/alignedlayer_deployment_output.json" +eigen_layer_deployment_config_file_path: "../contracts/script/output/devnet/eigenlayer_deployment_output.json" +eth_rpc_url: "http://localhost:8545" +eth_rpc_url_fallback: "http://localhost:8545" +eth_ws_url: "ws://localhost:8545" +eth_ws_url_fallback: "ws://localhost:8545" +eigen_metrics_ip_port_address: "localhost:9090" + +## ECDSA Configurations +ecdsa: + private_key_store_path: "../config-files/anvil.aggregator.ecdsa.key.json" + private_key_store_password: "" + +## BLS Configurations +bls: + private_key_store_path: "../config-files/anvil.aggregator.bls.key.json" + private_key_store_password: "" + +# ## Batcher configurations # batcher: +# block_interval: 3 +# batch_size_interval: 10 +# max_proof_size: 67108864 # 64 MiB +# max_batch_size: 268435456 # 256 MiB +# eth_ws_reconnects: 99999999999999 +# pre_verification_is_enabled: true + +## Aggregator Configurations +aggregator: + server_ip_port_address: localhost:8090 + bls_public_key_compendium_address: 0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44 + avs_service_manager_address: 0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690 + enable_metrics: false + metrics_ip_port_address: localhost:9091 + telemetry_ip_port_address: localhost:4001 +## Operator Configurations +# operator: +# aggregator_rpc_server_ip_port_address: localhost:8090 +# address: 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 +# earnings_receiver_address: 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 +# delegation_approver_address: "0x0000000000000000000000000000000000000000" +# staker_opt_out_window_blocks: 0 +# metadata_url: "https://yetanotherco.github.io/operator_metadata/metadata.json" +# enable_metrics: true +# metrics_ip_port_address: localhost:9092 +# max_batch_size: 268435456 # 256 MiB +# # Operators variables needed for register it in EigenLayer +# el_delegation_manager_address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" +# private_key_store_path: config-files/anvil.ecdsa.key.json +# bls_private_key_store_path: config-files/anvil.bls.key.json +# signer_type: local_keystore +# chain_id: 31337 From 4df06bf145a81af8d4d8db7e3e2abe54a17add6d Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 24 Oct 2024 13:01:52 -0300 Subject: [PATCH 050/135] make visibility of retry functions public --- core/chainio/avs_subscriber.go | 48 +++++++++++++++++----------------- core/chainio/avs_writer.go | 18 ++++++------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 47c94fd1f..88a34cff0 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -71,14 +71,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan // Subscribe to new tasks // Retry() - sub, err := subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err } // Retry() - subFallback, err := subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err @@ -121,7 +121,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() // Retry() - sub, err = subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err = SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err } @@ -129,7 +129,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() // Retry() - subFallback, err = subscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err = SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err } @@ -140,20 +140,20 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan return errorChannel, nil } -func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) (chan error, error) { +func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) (chan error, error) { // Create a new channel to receive new tasks internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) // Subscribe to new tasks // Retry() - sub, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err } // Retry() - subFallback, err := subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) return nil, err @@ -196,7 +196,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() // Retry() - sub, err = subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err = SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err } @@ -204,7 +204,7 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3Retryable(newTaskCreatedChan chan * s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() // Retry() - subFallback, err = subscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err = SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err } @@ -271,7 +271,7 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { // Retry() - latestBlock, err := s.blockNumberRetryable(context.Background()) + latestBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return nil, err } @@ -285,7 +285,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag } // Retry() - logs, err := s.filterBatchV2Retryable(fromBlock, context.Background()) + logs, err := s.FilterBatchV2Retryable(fromBlock, context.Background()) if err != nil { return nil, err } @@ -308,7 +308,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) // Retry() - state, err := s.batchesStateRetryable(nil, batchIdentifierHash) + state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -323,7 +323,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { // Retry() - latestBlock, err := s.blockNumberRetryable(context.Background()) + latestBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return nil, err } @@ -337,7 +337,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag } // Retry() - logs, err := s.filterBatchV3Retryable(fromBlock, context.Background()) + logs, err := s.FilterBatchV3Retryable(fromBlock, context.Background()) if err != nil { return nil, err } @@ -360,7 +360,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) // Retry() - state, err := s.batchesStateRetryable(nil, batchIdentifierHash) + state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -374,7 +374,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // Retry() - currentBlock, err := s.blockNumberRetryable(context.Background()) + currentBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return err } @@ -383,7 +383,7 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // Subscribe to new head c := make(chan *types.Header) // Retry() - sub, err := s.subscribeNewHeadRetryable(context.Background(), c) + sub, err := s.SubscribeNewHeadRetryable(context.Background(), c) if err != nil { return err } @@ -413,7 +413,7 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // |---RETRYABLE---| -func (s *AvsSubscriber) blockNumberRetryable(ctx context.Context) (*uint64, error) { +func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (*uint64, error) { latestBlock_func := func() (*uint64, error) { latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) if err != nil { @@ -427,7 +427,7 @@ func (s *AvsSubscriber) blockNumberRetryable(ctx context.Context) (*uint64, erro return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (s *AvsSubscriber) filterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { +func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) @@ -435,7 +435,7 @@ func (s *AvsSubscriber) filterBatchV2Retryable(fromBlock uint64, ctx context.Con return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (s *AvsSubscriber) filterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { +func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) @@ -443,7 +443,7 @@ func (s *AvsSubscriber) filterBatchV3Retryable(fromBlock uint64, ctx context.Con return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (s *AvsSubscriber) batchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (*struct { +func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (*struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int @@ -460,7 +460,7 @@ func (s *AvsSubscriber) batchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (s *AvsSubscriber) subscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (*ethereum.Subscription, error) { +func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (*ethereum.Subscription, error) { subscribeNewHead_func := func() (*ethereum.Subscription, error) { sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) if err != nil { @@ -474,7 +474,7 @@ func (s *AvsSubscriber) subscribeNewHeadRetryable(ctx context.Context, c chan<- return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func subscribeToNewTasksV2Retrayable( +func SubscribeToNewTasksV2Retrayable( serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, logger sdklogging.Logger, @@ -491,7 +491,7 @@ func subscribeToNewTasksV2Retrayable( return *sub, nil } -func subscribeToNewTasksV3Retryable( +func SubscribeToNewTasksV3Retryable( serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, logger sdklogging.Logger, diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index b8404316b..2df7617c6 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -75,7 +75,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction // Retry - tx, err := w.respondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err := w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err } @@ -89,7 +89,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts.NoSend = false txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit // Retry - tx, err = w.respondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err = w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bin // Get RespondToTaskFeeLimit // Retry - batchState, err := w.batchesStateRetryable(batchIdentifierHash) + batchState, err := w.BatchesStateRetryable(batchIdentifierHash) if err != nil { // Fallback also failed // Proceed to check values against simulated costs @@ -145,7 +145,7 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress // TODO(pat): Separate Retry from call // Retry - aggregatorBalance, err := w.balanceAtRetryable(ctx, aggregatorAddress, nil) + aggregatorBalance, err := w.BalanceAtRetryable(ctx, aggregatorAddress, nil) if err != nil { // Ignore and continue. w.logger.Error("failed to get aggregator balance: %v", err) @@ -162,7 +162,7 @@ func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byt // Get batcher balance // TODO(pat): Separate Retry from call // Retry - batcherBalance, err := w.batcherBalancesRetryable(senderAddress) + batcherBalance, err := w.BatcherBalancesRetryable(senderAddress) if err != nil { // Ignore and continue. w.logger.Error("Failed to get batcherBalance", "error", err) @@ -177,7 +177,7 @@ func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byt // |---RETRYABLE---| -func (w *AvsWriter) respondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { +func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { respondToTaskV2_func := func() (*types.Transaction, error) { tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { @@ -192,7 +192,7 @@ func (w *AvsWriter) respondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl return connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (w *AvsWriter) batchesStateRetryable(arg0 [32]byte) (struct { +func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int @@ -217,7 +217,7 @@ func (w *AvsWriter) batchesStateRetryable(arg0 [32]byte) (struct { } // TODO: separate retry concerns and inline them in real functions -func (w *AvsWriter) batcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { +func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { batcherBalances_func := func() (*big.Int, error) { batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) if err != nil { @@ -231,7 +231,7 @@ func (w *AvsWriter) batcherBalancesRetryable(senderAddress common.Address) (*big return connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (w *AvsWriter) balanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { +func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { balanceAt_func := func() (*big.Int, error) { aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) From 21ca36b64fecbcb4628c5c6b93288b212fa33d74 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 24 Oct 2024 13:02:07 -0300 Subject: [PATCH 051/135] finish mocks for the rest of the tests --- core/connection_test.go | 454 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 420 insertions(+), 34 deletions(-) diff --git a/core/connection_test.go b/core/connection_test.go index 3cbf38114..1308036d8 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -13,12 +13,18 @@ import ( "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" + "github.com/Layr-Labs/eigensdk-go/crypto/bls" rpccalls "github.com/Layr-Labs/eigensdk-go/metrics/collectors/rpc_calls" + eigentypes "github.com/Layr-Labs/eigensdk-go/types" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" + aggregator "github.com/yetanotherco/aligned_layer/aggregator/pkg" + servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" connection "github.com/yetanotherco/aligned_layer/core" + "github.com/yetanotherco/aligned_layer/core/chainio" + "github.com/yetanotherco/aligned_layer/core/config" "github.com/yetanotherco/aligned_layer/core/utils" ) @@ -71,8 +77,10 @@ func SetupAnvil(port uint16) (*exec.Cmd, *eth.InstrumentedClient, error) { http_rpc_url := fmt.Sprintf("http://localhost:%d", port) // Create a command - cmd := exec.Command("anvil", "--port", port_str, "--load-state", "../contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json", "--block-time", "7") - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd := exec.Command("anvil", "--port", port_str, "--load-state", "../contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json", "--block-time", "3") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } // Run the command err = cmd.Start() @@ -154,7 +162,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - // Assert Call succeeds why Anvil running + // Assert Call succeeds when Anvil running _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) if err.Error() != "not found" { @@ -164,17 +172,18 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { // Kill Anvil if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + fmt.Printf("error killing process: %v\n", err) return } + time.Sleep(2 * time.Second) - // Fails with "not found" + // Errors out but "not found" receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) assert.Nil(t, receipt, "Receipt not empty") assert.NotEqual(t, err.Error(), "not found") // Start anvil - _, client, err = SetupAnvil(8545) + cmd, client, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } @@ -184,71 +193,448 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { if err.Error() != "not found" { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) } -} -/* - -func TestSendAggregatedResponseRetryable(t *testing.T) { + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } func TestInitializeNewTaskRetryable(t *testing.T) { - //TODO: Instantiate Aggregator + + //Start Anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + //Start Aggregator + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + agg, err := aggregator.NewAggregator(*aggregatorConfig) + if err != nil { + aggregatorConfig.BaseConfig.Logger.Error("Cannot create aggregator", "err", err) + return + } + quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(byte(0))} + quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(byte(57))} + + // Should succeed with err msg + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.Nil(t, err) + // TODO: Find exact error to assert + + // Kill Anvil + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) + + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + // Start Anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + // Should succeed + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + if err = cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } // |--Server Retry Tests--| func TestProcessNewSignatureRetryable(t *testing.T) { - agg := NewAggregator() - agg.ProcessNewSignatureRetryable() + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + //Start Aggregator + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + agg, err := aggregator.NewAggregator(*aggregatorConfig) + if err != nil { + aggregatorConfig.BaseConfig.Logger.Error("Cannot create aggregator", "err", err) + return + } + zero_bytes := [32]byte{} + zero_sig := bls.NewZeroSignature() + eigen_bytes := eigentypes.Bytes32{} + + err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + assert.Nil(t, err) + // TODO: Find exact error to assert + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) + /* + + err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + assert.NotNil(t, err) + fmt.Printf("Error Processing New Signature: %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + assert.Nil(t, err) + fmt.Printf("Error Processing New Signature: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + */ } -// |--Subscriber Retry Tests--| +// |--AVS-Subscriber Retry Tests--| func TestSubscribeToNewTasksV3Retryable(t *testing.T) { - newBatchChan := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } - baseConfig := core.NewBaseConfig("") - avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(baseConfig) + channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + baseConfig := aggregatorConfig.BaseConfig + s, err := chainio.NewAvsServiceBindings( + baseConfig.AlignedLayerDeploymentConfig.AlignedLayerServiceManagerAddr, + baseConfig.AlignedLayerDeploymentConfig.AlignedLayerOperatorStateRetrieverAddr, + baseConfig.EthWsClient, baseConfig.EthWsClientFallback, baseConfig.Logger) if err != nil { - return nil, err + fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) } - agg.taskSubscriber, err = avsSubscriber.SubscribeToNewTasksV3Retryable(newBatchChan) + fmt.Printf("Subscribing") + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.Nil(t, err) + // TODO: Find exact error to assert + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) + + /* + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + */ } -// |--AVS-Writer Retry Tests--| +func TestSubscribeToNewTasksV2(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } -func TestRespondToTaskV2(t *testing.T) { -} + channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + baseConfig := aggregatorConfig.BaseConfig + s, err := chainio.NewAvsServiceBindings( + baseConfig.AlignedLayerDeploymentConfig.AlignedLayerServiceManagerAddr, + baseConfig.AlignedLayerDeploymentConfig.AlignedLayerOperatorStateRetrieverAddr, + baseConfig.EthWsClient, baseConfig.EthWsClientFallback, baseConfig.Logger) + if err != nil { + fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) + } -func TestBatchesStateWriter(t *testing.T) { -} + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.Nil(t, err) + // TODO: Find exact error to assert -func TestBalanceAt(t *testing.T) { + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) + + /* + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + */ } -func TestBatchersBalances(t *testing.T) { -} +func TestBlockNumber(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } -// |--AVS-Subscriber Retry Tests--| + //channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + sub, err := chainio.NewAvsSubscriberFromConfig(aggregatorConfig.BaseConfig) + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err, "Failed to Retrieve Block Number") -func TestSubscribeToNewTasksV2(t *testing.T) { + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } -func TestSubscribeToNewTasksV3(t *testing.T) { -} +func TestFilterBatchV2(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } -func TestBlockNumber(t *testing.T) { -} + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(aggregatorConfig.BaseConfig) + if err != nil { + return + } + _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) + assert.Nil(t, err, "Failed to Retrieve Block Number") + // TODO: Find exact error to assert -func TestFilterBatchV2(t *testing.T) { + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } func TestFilterBatchV3(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(aggregatorConfig.BaseConfig) + if err != nil { + return + } + _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) + //TODO: Find error to assert + assert.NotNil(t, err, "Succeeded in filtering logs") + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } func TestBatchesStateSubscriber(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(aggregatorConfig.BaseConfig) + if err != nil { + return + } + + zero_bytes := [32]byte{} + _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) + //TODO: Find exact failure error + assert.NotNil(t, err, "BatchesState") + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } func TestSubscribeNewHead(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + c := make(chan *types.Header) + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + avsSubscriber, err := chainio.NewAvsSubscriberFromConfig(aggregatorConfig.BaseConfig) + if err != nil { + return + } + + avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) + assert.Nil(t, err, "Should be 0") + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) +} + +/* +// |--AVS-Writer Retry Tests--| + +func TestRespondToTaskV2(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + w, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) + txOpts := *w.Signer.GetTxOpts() + aggregator_address := common.HexToAddress("0x0") + zero_bytes := [32]byte{} + + w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address) + assert.Nil(t, err, "Should be 0") + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } */ + +func TestBatchesStateWriter(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + avsWriter, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) + if err != nil { + return + } + zero_bytes := [32]byte{} + + _, err = avsWriter.BatchesStateRetryable(zero_bytes) + assert.Nil(t, err, "Should be 0") + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) +} + +func TestBalanceAt(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + avsWriter, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) + if err != nil { + return + } + //TODO: Source Aggregator Address + aggregator_address := common.HexToAddress("0x0") + + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(0)) + assert.Nil(t, err, "Should be 0") + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) +} + +func TestBatchersBalances(t *testing.T) { + // Start anvil + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + avsWriter, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) + if err != nil { + return + } + //TODO: Source real one + sender_address := common.HexToAddress("0x0") + + _, err = avsWriter.BatcherBalancesRetryable(sender_address) + assert.Nil(t, err, "Should be 0") + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) +} From 347f1556e416f1fadfe98edc3968867b22e2b7d1 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 24 Oct 2024 13:02:32 -0300 Subject: [PATCH 052/135] add missing retry --- core/connection_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/connection_test.go b/core/connection_test.go index 1308036d8..e05a0ea26 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -424,6 +424,9 @@ func TestBlockNumber(t *testing.T) { //channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") sub, err := chainio.NewAvsSubscriberFromConfig(aggregatorConfig.BaseConfig) + if err != nil { + return + } _, err = sub.BlockNumberRetryable(context.Background()) assert.Nil(t, err, "Failed to Retrieve Block Number") From 330c49bef4d5c058df24d8130e796945850e5458 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 24 Oct 2024 14:18:30 -0300 Subject: [PATCH 053/135] test accept aggregator init function --- core/connection_test.go | 87 +++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 46 deletions(-) diff --git a/core/connection_test.go b/core/connection_test.go index e05a0ea26..ad366abd5 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -13,14 +13,11 @@ import ( "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" - "github.com/Layr-Labs/eigensdk-go/crypto/bls" rpccalls "github.com/Layr-Labs/eigensdk-go/metrics/collectors/rpc_calls" - eigentypes "github.com/Layr-Labs/eigensdk-go/types" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - aggregator "github.com/yetanotherco/aligned_layer/aggregator/pkg" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/chainio" @@ -177,31 +174,34 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { } time.Sleep(2 * time.Second) - // Errors out but "not found" - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - assert.Nil(t, receipt, "Receipt not empty") - assert.NotEqual(t, err.Error(), "not found") + /* + // Errors out but "not found" + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + assert.Nil(t, receipt, "Receipt not empty") + assert.NotEqual(t, err.Error(), "not found") - // Start anvil - cmd, client, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } + // Start anvil + cmd, client, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } - _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - assert.NotNil(t, err, "Call to Anvil failed") - if err.Error() != "not found" { - fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) - } + _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + assert.NotNil(t, err, "Call to Anvil failed") + if err.Error() != "not found" { + fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) + } - // Kill Anvil at end of test - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - time.Sleep(2 * time.Second) + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) + */ } +/* func TestInitializeNewTaskRetryable(t *testing.T) { //Start Anvil @@ -232,28 +232,24 @@ func TestInitializeNewTaskRetryable(t *testing.T) { } time.Sleep(2 * time.Second) - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) - - // Start Anvil - cmd, _, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) - // Should succeed - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + // Start Anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } - if err = cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - time.Sleep(2 * time.Second) + // Should succeed + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) } +*/ +/* // |--Server Retry Tests--| func TestProcessNewSignatureRetryable(t *testing.T) { // Start anvil @@ -283,7 +279,6 @@ func TestProcessNewSignatureRetryable(t *testing.T) { return } time.Sleep(2 * time.Second) - /* err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.NotNil(t, err) @@ -304,8 +299,8 @@ func TestProcessNewSignatureRetryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - */ } +*/ // |--AVS-Subscriber Retry Tests--| @@ -476,7 +471,7 @@ func TestFilterBatchV3(t *testing.T) { } _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) //TODO: Find error to assert - assert.NotNil(t, err, "Succeeded in filtering logs") + assert.Nil(t, err, "Succeeded in filtering logs") // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -502,7 +497,7 @@ func TestBatchesStateSubscriber(t *testing.T) { zero_bytes := [32]byte{} _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) //TODO: Find exact failure error - assert.NotNil(t, err, "BatchesState") + assert.Nil(t, err, "BatchesState") // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -605,7 +600,7 @@ func TestBalanceAt(t *testing.T) { //TODO: Source Aggregator Address aggregator_address := common.HexToAddress("0x0") - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(0)) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(16)) assert.Nil(t, err, "Should be 0") // Kill Anvil at end of test From f0e58cf1545c566d0b4e61af1167561656bdf8b5 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Fri, 25 Oct 2024 15:56:41 -0300 Subject: [PATCH 054/135] fix: permanent err in send aggregator --- core/chainio/avs_writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 9f91b5f9b..1e37e9c42 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -118,7 +118,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // Retry with fallback gasPrice, err = w.ClientFallback.SuggestGasPrice(context.Background()) if err != nil { - return nil, fmt.Errorf("transaction simulation failed: %v", err) + return nil, connection.PermanentError{Inner: err} } } txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, GasBumpPercentage, i) From 85506755c81511829c66fbf4c637c5a9a4f33aec Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 25 Oct 2024 16:51:38 -0300 Subject: [PATCH 055/135] progress --- core/chainio/avs_subscriber.go | 2 +- core/connection_test.go | 308 +++++++++++++++++++++++++-------- operator/pkg/operator.go | 4 +- 3 files changed, 239 insertions(+), 75 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 88a34cff0..71508bb81 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -65,7 +65,7 @@ func NewAvsSubscriberFromConfig(baseConfig *config.BaseConfig) (*AvsSubscriber, }, nil } -func (s *AvsSubscriber) SubscribeToNewTasksV2Retrayable(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) (chan error, error) { +func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) (chan error, error) { // Create a new channel to receive new tasks internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) diff --git a/core/connection_test.go b/core/connection_test.go index ad366abd5..09c05532d 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -86,7 +86,7 @@ func SetupAnvil(port uint16) (*exec.Cmd, *eth.InstrumentedClient, error) { } // Delay needed for anvil to start - time.Sleep(1 * time.Second) + time.Sleep(500 * time.Millisecond) reg := prometheus.NewRegistry() rpcCallsCollector := rpccalls.NewCollector("ethRpc", reg) @@ -172,33 +172,31 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) - /* - // Errors out but "not found" - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - assert.Nil(t, receipt, "Receipt not empty") - assert.NotEqual(t, err.Error(), "not found") + // Errors out but "not found" + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + assert.Nil(t, receipt, "Receipt not empty") + assert.NotEqual(t, err.Error(), "not found") - // Start anvil - cmd, client, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } + // Start anvil + cmd, client, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } - _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) - assert.NotNil(t, err, "Call to Anvil failed") - if err.Error() != "not found" { - fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) - } + _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + assert.NotNil(t, err, "Call to Anvil failed") + if err.Error() != "not found" { + fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) + } - // Kill Anvil at end of test - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - time.Sleep(2 * time.Second) - */ + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } /* @@ -332,29 +330,28 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) - /* - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) - assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) - // Start anvil - cmd, _, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) - assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) - // Kill Anvil at end of test - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - */ + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestSubscribeToNewTasksV2(t *testing.T) { @@ -364,7 +361,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) + channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") baseConfig := aggregatorConfig.BaseConfig s, err := chainio.NewAvsServiceBindings( @@ -375,7 +372,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.Nil(t, err) // TODO: Find exact error to assert @@ -384,29 +381,28 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) - /* - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) - assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) - // Start anvil - cmd, _, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) - assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) - // Kill Anvil at end of test - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - */ + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestBlockNumber(t *testing.T) { @@ -430,7 +426,28 @@ func TestBlockNumber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestFilterBatchV2(t *testing.T) { @@ -454,7 +471,28 @@ func TestFilterBatchV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestFilterBatchV3(t *testing.T) { @@ -478,7 +516,28 @@ func TestFilterBatchV3(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestBatchesStateSubscriber(t *testing.T) { @@ -504,7 +563,28 @@ func TestBatchesStateSubscriber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestSubscribeNewHead(t *testing.T) { @@ -529,7 +609,28 @@ func TestSubscribeNewHead(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } /* @@ -582,7 +683,28 @@ func TestBatchesStateWriter(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestBalanceAt(t *testing.T) { @@ -608,7 +730,28 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } func TestBatchersBalances(t *testing.T) { @@ -634,5 +777,26 @@ func TestBatchersBalances(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = sub.BlockNumberRetryable(context.Background()) + assert.Nil(t, err) + fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(500 * time.Millisecond) } diff --git a/operator/pkg/operator.go b/operator/pkg/operator.go index eaf9eed2b..8a96b51c4 100644 --- a/operator/pkg/operator.go +++ b/operator/pkg/operator.go @@ -153,11 +153,11 @@ func NewOperatorFromConfig(configuration config.OperatorConfig) (*Operator, erro } func (o *Operator) SubscribeToNewTasksV2() (chan error, error) { - return o.avsSubscriber.SubscribeToNewTasksV2Retrayable(o.NewTaskCreatedChanV2) + return o.avsSubscriber.SubscribeToNewTasksV2(o.NewTaskCreatedChanV2) } func (o *Operator) SubscribeToNewTasksV3() (chan error, error) { - return o.avsSubscriber.SubscribeToNewTasksV3Retryable(o.NewTaskCreatedChanV3) + return o.avsSubscriber.SubscribeToNewTasksV3(o.NewTaskCreatedChanV3) } type OperatorLastProcessedBatch struct { From e5de508efac829cc6c7a11f24a046c743a6dfdb4 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 28 Oct 2024 17:42:01 -0300 Subject: [PATCH 056/135] test fixes --- core/connection.go | 10 +- core/connection_test.go | 240 ++++++++++++++++++++++++++-------------- 2 files changed, 163 insertions(+), 87 deletions(-) diff --git a/core/connection.go b/core/connection.go index 6c84e49d7..c0d9926f2 100644 --- a/core/connection.go +++ b/core/connection.go @@ -27,7 +27,15 @@ const NumRetries = 3 func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { i := 0 f := func() (*T, error) { - val, err := functionToRetry() + // Create a channel to receive results from the protected function call + var val *T + var err error + + defer func() { + if r := recover(); r != nil { + } + val, err := functionToRetry() + }() i++ if perm, ok := err.(PermanentError); err != nil && ok { return nil, backoff.Permanent(perm.Inner) diff --git a/core/connection_test.go b/core/connection_test.go index 09c05532d..87314ac25 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -8,16 +8,20 @@ import ( "os" "os/exec" "strconv" + "strings" "syscall" "testing" "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" + "github.com/Layr-Labs/eigensdk-go/crypto/bls" rpccalls "github.com/Layr-Labs/eigensdk-go/metrics/collectors/rpc_calls" + eigentypes "github.com/Layr-Labs/eigensdk-go/types" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" + aggregator "github.com/yetanotherco/aligned_layer/aggregator/pkg" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/chainio" @@ -86,7 +90,7 @@ func SetupAnvil(port uint16) (*exec.Cmd, *eth.InstrumentedClient, error) { } // Delay needed for anvil to start - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) reg := prometheus.NewRegistry() rpcCallsCollector := rpccalls.NewCollector("ethRpc", reg) @@ -172,7 +176,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) // Errors out but "not found" receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) @@ -196,10 +200,9 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } -/* func TestInitializeNewTaskRetryable(t *testing.T) { //Start Anvil @@ -230,24 +233,22 @@ func TestInitializeNewTaskRetryable(t *testing.T) { } time.Sleep(2 * time.Second) - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) - - // Start Anvil - cmd, _, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } - - // Should succeed - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + // Start Anvil + _, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + // Should succeed + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) } -*/ -/* // |--Server Retry Tests--| func TestProcessNewSignatureRetryable(t *testing.T) { // Start anvil @@ -278,27 +279,26 @@ func TestProcessNewSignatureRetryable(t *testing.T) { } time.Sleep(2 * time.Second) - err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) - assert.NotNil(t, err) - fmt.Printf("Error Processing New Signature: %s\n", err) - - // Start anvil - cmd, _, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } - - err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) - assert.Nil(t, err) - fmt.Printf("Error Processing New Signature: %s\n", err) - - // Kill Anvil at end of test - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } + err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + assert.NotNil(t, err) + fmt.Printf("Error Processing New Signature: %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + assert.Nil(t, err) + fmt.Printf("Error Processing New Signature: %s\n", err) + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } } -*/ // |--AVS-Subscriber Retry Tests--| @@ -330,7 +330,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) @@ -351,7 +351,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestSubscribeToNewTasksV2(t *testing.T) { @@ -381,7 +381,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) @@ -402,7 +402,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestBlockNumber(t *testing.T) { @@ -426,7 +426,7 @@ func TestBlockNumber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) @@ -447,7 +447,7 @@ func TestBlockNumber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestFilterBatchV2(t *testing.T) { @@ -471,9 +471,9 @@ func TestFilterBatchV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.NotNil(t, err) fmt.Printf(": %s\n", err) @@ -483,7 +483,7 @@ func TestFilterBatchV2(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.Nil(t, err) fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) @@ -492,7 +492,7 @@ func TestFilterBatchV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestFilterBatchV3(t *testing.T) { @@ -516,9 +516,9 @@ func TestFilterBatchV3(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) assert.NotNil(t, err) fmt.Printf(": %s\n", err) @@ -528,7 +528,7 @@ func TestFilterBatchV3(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) assert.Nil(t, err) fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) @@ -537,7 +537,7 @@ func TestFilterBatchV3(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestBatchesStateSubscriber(t *testing.T) { @@ -563,9 +563,9 @@ func TestBatchesStateSubscriber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) fmt.Printf(": %s\n", err) @@ -575,7 +575,7 @@ func TestBatchesStateSubscriber(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.Nil(t, err) fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) @@ -584,7 +584,7 @@ func TestBatchesStateSubscriber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestSubscribeNewHead(t *testing.T) { @@ -609,9 +609,9 @@ func TestSubscribeNewHead(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) fmt.Printf(": %s\n", err) @@ -621,7 +621,7 @@ func TestSubscribeNewHead(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) @@ -630,10 +630,9 @@ func TestSubscribeNewHead(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } -/* // |--AVS-Writer Retry Tests--| func TestRespondToTaskV2(t *testing.T) { @@ -643,23 +642,83 @@ func TestRespondToTaskV2(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } + g2Point := servicemanager.BN254G2Point{ + X: [2]*big.Int{big.NewInt(4), big.NewInt(3)}, + Y: [2]*big.Int{big.NewInt(2), big.NewInt(8)}, + } + + g1Point := servicemanager.BN254G1Point{ + X: big.NewInt(2), + Y: big.NewInt(1), + } + + g1Points := []servicemanager.BN254G1Point{ + g1Point, g1Point, g1Point, + } + + // Or if you want to initialize with specific values + + nonSignerStakesAndSignature := servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature{ + NonSignerPubkeys: g1Points, + QuorumApks: g1Points, + ApkG2: g2Point, + Sigma: g1Point, + NonSignerQuorumBitmapIndices: make([]uint32, 3), + QuorumApkIndices: make([]uint32, 3), + TotalStakeIndices: make([]uint32, 3), + NonSignerStakeIndices: make([][]uint32, 1, 3), + } + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") w, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) + if err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } txOpts := *w.Signer.GetTxOpts() - aggregator_address := common.HexToAddress("0x0") + aggregator_address := common.HexToAddress("0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690") zero_bytes := [32]byte{} - w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address) - assert.Nil(t, err, "Should be 0") + // Panics if 0 object passed + _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + assert.NotNil(t, err, "Should error") + // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" + if !strings.Contains(err.Error(), "execution reverted: custom error 0x2396d34e:") { + t.Errorf("Respond to task V2 Retryable did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") + return + } // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) + + _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + + // Start anvil + cmd, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + assert.NotNil(t, err, "Should error") + // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" + if !strings.Contains(err.Error(), "execution reverted: custom error 0x2396d34e:") { + t.Errorf("Respond to task V2 Retryable did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") + return + } + + // Kill Anvil at end of test + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(750 * time.Millisecond) } -*/ func TestBatchesStateWriter(t *testing.T) { // Start anvil @@ -671,11 +730,15 @@ func TestBatchesStateWriter(t *testing.T) { aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") avsWriter, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) if err != nil { + fmt.Printf("error killing process: %v\n", err) return } - zero_bytes := [32]byte{} + num := big.NewInt(6) - _, err = avsWriter.BatchesStateRetryable(zero_bytes) + var bytes [32]byte + num.FillBytes(bytes[:]) + + _, err = avsWriter.BatchesStateRetryable(bytes) assert.Nil(t, err, "Should be 0") // Kill Anvil at end of test @@ -683,11 +746,14 @@ func TestBatchesStateWriter(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) - _, err = sub.BlockNumberRetryable(context.Background()) - assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Directly panics -> Need to catch in retry. + /* + _, err = avsWriter.BatchesStateRetryable(bytes) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + */ // Start anvil cmd, _, err = SetupAnvil(8545) @@ -695,7 +761,7 @@ func TestBatchesStateWriter(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsWriter.BatchesStateRetryable(bytes) assert.Nil(t, err) fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) @@ -704,7 +770,7 @@ func TestBatchesStateWriter(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestBalanceAt(t *testing.T) { @@ -730,9 +796,9 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(16)) assert.NotNil(t, err) fmt.Printf(": %s\n", err) @@ -742,7 +808,7 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(16)) assert.Nil(t, err) fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) @@ -751,7 +817,7 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } func TestBatchersBalances(t *testing.T) { @@ -777,11 +843,13 @@ func TestBatchersBalances(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) - _, err = sub.BlockNumberRetryable(context.Background()) - assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + /* + _, err = avsWriter.BatcherBalancesRetryable(sender_address) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) + */ // Start anvil cmd, _, err = SetupAnvil(8545) @@ -789,7 +857,7 @@ func TestBatchersBalances(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = avsWriter.BatcherBalancesRetryable(sender_address) assert.Nil(t, err) fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) @@ -798,5 +866,5 @@ func TestBatchersBalances(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(500 * time.Millisecond) + time.Sleep(750 * time.Millisecond) } From b40f5a0d1671769d139a5d095a21730ff0d6348e Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 28 Oct 2024 19:48:08 -0300 Subject: [PATCH 057/135] Comment out aggregator tests as they panic due to known error in eigen-sdk --- core/connection.go | 44 +++++++++++++++++++------- core/connection_test.go | 68 ++++++++++++++++++++++------------------- 2 files changed, 70 insertions(+), 42 deletions(-) diff --git a/core/connection.go b/core/connection.go index c0d9926f2..d9e6a51e2 100644 --- a/core/connection.go +++ b/core/connection.go @@ -1,6 +1,7 @@ package connection import ( + "fmt" "time" "github.com/cenkalti/backoff/v4" @@ -19,6 +20,19 @@ func (e PermanentError) Is(err error) bool { return ok } +type TransientError struct { + Inner error +} + +func (e TransientError) Error() string { return e.Inner.Error() } +func (e TransientError) Unwrap() error { + return e.Inner +} +func (e TransientError) Is(err error) bool { + _, ok := err.(TransientError) + return ok +} + const MinDelay = 1000 const RetryFactor = 2 const NumRetries = 3 @@ -27,19 +41,27 @@ const NumRetries = 3 func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { i := 0 f := func() (*T, error) { - // Create a channel to receive results from the protected function call - var val *T - var err error - - defer func() { - if r := recover(); r != nil { + var ( + val *T + err error + ) + func() { + // I + defer func() { + if r := recover(); r != nil { + if panic_err, ok := r.(error); ok { + err = TransientError{panic_err} + } else { + err = TransientError{fmt.Errorf("panicked: %v", panic_err)} + } + } + }() + val, err = functionToRetry() + i++ + if perm, ok := err.(PermanentError); err != nil && ok { + err = backoff.Permanent(perm.Inner) } - val, err := functionToRetry() }() - i++ - if perm, ok := err.(PermanentError); err != nil && ok { - return nil, backoff.Permanent(perm.Inner) - } return val, err } diff --git a/core/connection_test.go b/core/connection_test.go index 87314ac25..464645abb 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -14,14 +14,11 @@ import ( "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" - "github.com/Layr-Labs/eigensdk-go/crypto/bls" rpccalls "github.com/Layr-Labs/eigensdk-go/metrics/collectors/rpc_calls" - eigentypes "github.com/Layr-Labs/eigensdk-go/types" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - aggregator "github.com/yetanotherco/aligned_layer/aggregator/pkg" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/chainio" @@ -153,8 +150,6 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { Data: []byte{0x11, 0x11, 0x11}, }) - ctx := context.WithoutCancel(context.Background()) - hash := tx.Hash() // Start anvil @@ -164,7 +159,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { } // Assert Call succeeds when Anvil running - _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) if err.Error() != "not found" { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) @@ -179,7 +174,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { time.Sleep(750 * time.Millisecond) // Errors out but "not found" - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + receipt, err := utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.Nil(t, receipt, "Receipt not empty") assert.NotEqual(t, err.Error(), "not found") @@ -189,7 +184,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = utils.WaitForTransactionReceiptRetryable(*client, ctx, hash) + _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err, "Call to Anvil failed") if err.Error() != "not found" { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) @@ -203,10 +198,11 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { time.Sleep(750 * time.Millisecond) } +/* func TestInitializeNewTaskRetryable(t *testing.T) { //Start Anvil - cmd, _, err := SetupAnvil(8545) + _, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } @@ -226,28 +222,37 @@ func TestInitializeNewTaskRetryable(t *testing.T) { assert.Nil(t, err) // TODO: Find exact error to assert - // Kill Anvil - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - time.Sleep(2 * time.Second) - - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) - - // Start Anvil - _, _, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } - - // Should succeed - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + // Kill Anvil + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) + + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + // Start Anvil + _, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + // Should succeed + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + // Kill Anvil + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + time.Sleep(2 * time.Second) } +*/ + +/* // |--Server Retry Tests--| func TestProcessNewSignatureRetryable(t *testing.T) { @@ -269,7 +274,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { eigen_bytes := eigentypes.Bytes32{} err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) - assert.Nil(t, err) + assert.NotNil(t, err) // TODO: Find exact error to assert // Kill Anvil at end of test @@ -299,6 +304,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { return } } +*/ // |--AVS-Subscriber Retry Tests--| From 2b25d45192a7a5622947a8377bf0a0f4dbcc498f Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 29 Oct 2024 16:10:56 -0300 Subject: [PATCH 058/135] fix: address review comments --- core/chainio/avs_writer.go | 6 ++++-- core/utils/eth_client_utils.go | 23 +++++++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 1e37e9c42..edcf90d38 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -81,7 +81,10 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E // Sends AggregatedResponse and waits for the receipt for three blocks, if not received // it will try again bumping the last tx gas price based on `CalculateGasPriceBump` // This process happens indefinitely until the transaction is included. -// Note: If the rpc endpoints fail, the retry will stop, as it will infinitely try +// +// Note: If the rpc endpoints fail, the retry will stop it returning a permanent error. +// This is because the retries are infinite and we want to prevent increasing the time between them too much as it is exponential. +// And we might also if the rpc is down for a good period of time, we might fall into an infinite waiting. func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, onRetry func()) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction @@ -125,7 +128,6 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) - if err != nil { return nil, connection.PermanentError{Inner: err} } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index ec0abb8ac..f81561993 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -25,9 +25,8 @@ func WaitForTransactionReceipt(client eth.InstrumentedClient, ctx context.Contex // if context has timed out, return if ctx.Err() != nil { return nil, ctx.Err() - } else { - time.Sleep(sleepTime) } + time.Sleep(sleepTime) } return nil, fmt.Errorf("transaction receipt not found for txHash: %s", txHash.String()) } @@ -48,13 +47,21 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e return quorumThresholdPercentages } -// Very basic algorithm to calculate the gasPrice bump based on the currentGasPrice a constant percentage and the retry number. -// It adds a the percentage to the current gas price and a 5% * i, where i is the iteration number. That is: -func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, percentage int, i int) *big.Int { - retryPercentage := new(big.Int).Mul(big.NewInt(5), big.NewInt(int64(i))) - percentageBump := new(big.Int).Add(big.NewInt(int64(percentage)), retryPercentage) - bumpAmount := new(big.Int).Mul(currentGasPrice, percentageBump) +// Simple algorithm to calculate the gasPrice bump based on: +// the currentGasPrice, a base bump percentage, and the retry count. +// Formula: currentGasPrice + (currentGasPrice * (baseBumpPercentage + retryCount * incrementalRetryPercentage) / 100) +func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercentage int, retryCount int) *big.Int { + // Incremental percentage increase for each retry attempt (5%) + incrementalRetryPercentage := new(big.Int).Mul(big.NewInt(5), big.NewInt(int64(retryCount))) + + // Total bump percentage: base bump + incremental percentage based on retry count + totalBumpPercentage := new(big.Int).Add(big.NewInt(int64(baseBumpPercentage)), incrementalRetryPercentage) + + // Calculate the bump amount: currentGasPrice * totalBumpPercentage / 100 + bumpAmount := new(big.Int).Mul(currentGasPrice, totalBumpPercentage) bumpAmount = new(big.Int).Div(bumpAmount, big.NewInt(100)) + + // Final bumped gas price: currentGasPrice + bumpAmount bumpedGasPrice := new(big.Int).Add(currentGasPrice, bumpAmount) return bumpedGasPrice From 696781bd33b841bf007c1ea5e6610f73bcfa1aa5 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 29 Oct 2024 16:39:18 -0300 Subject: [PATCH 059/135] save progress --- core/chainio/avs_writer.go | 4 +-- core/connection.go | 1 - core/connection_test.go | 46 +++++++++------------------------- core/utils/eth_client_utils.go | 23 +++++++++++++++-- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 2df7617c6..50fdefeff 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -206,8 +206,9 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { if err != nil { // Retry with fallback state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) + // If error is not nil throw out result in state! if err != nil { - return nil, err + return &state, err } } return &state, err @@ -216,7 +217,6 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { return *state, err } -// TODO: separate retry concerns and inline them in real functions func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { batcherBalances_func := func() (*big.Int, error) { batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) diff --git a/core/connection.go b/core/connection.go index d9e6a51e2..7a21908e4 100644 --- a/core/connection.go +++ b/core/connection.go @@ -46,7 +46,6 @@ func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, f err error ) func() { - // I defer func() { if r := recover(); r != nil { if panic_err, ok := r.(error); ok { diff --git a/core/connection_test.go b/core/connection_test.go index 464645abb..ad1d5ca68 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -161,6 +161,11 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { // Assert Call succeeds when Anvil running _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) + if _, ok := err.(*connection.TransientError); !ok { + t.Errorf("Retry on failure did not return backoff Permanent Error: %s", err) + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + + } if err.Error() != "not found" { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) return @@ -171,10 +176,10 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) // Errors out but "not found" receipt, err := utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + fmt.Printf("error fetching from killed anvil: %v\n", err) assert.Nil(t, receipt, "Receipt not empty") assert.NotEqual(t, err.Error(), "not found") @@ -195,7 +200,6 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } /* @@ -253,7 +257,6 @@ func TestInitializeNewTaskRetryable(t *testing.T) { */ /* - // |--Server Retry Tests--| func TestProcessNewSignatureRetryable(t *testing.T) { // Start anvil @@ -336,7 +339,6 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) @@ -357,7 +359,6 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestSubscribeToNewTasksV2(t *testing.T) { @@ -387,7 +388,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) @@ -408,7 +408,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestBlockNumber(t *testing.T) { @@ -432,7 +431,6 @@ func TestBlockNumber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) @@ -453,7 +451,6 @@ func TestBlockNumber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestFilterBatchV2(t *testing.T) { @@ -477,7 +474,6 @@ func TestFilterBatchV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.NotNil(t, err) @@ -498,7 +494,6 @@ func TestFilterBatchV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestFilterBatchV3(t *testing.T) { @@ -522,7 +517,6 @@ func TestFilterBatchV3(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) assert.NotNil(t, err) @@ -543,7 +537,6 @@ func TestFilterBatchV3(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestBatchesStateSubscriber(t *testing.T) { @@ -569,7 +562,6 @@ func TestBatchesStateSubscriber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) @@ -590,7 +582,6 @@ func TestBatchesStateSubscriber(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestSubscribeNewHead(t *testing.T) { @@ -615,7 +606,6 @@ func TestSubscribeNewHead(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) @@ -636,7 +626,6 @@ func TestSubscribeNewHead(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } // |--AVS-Writer Retry Tests--| @@ -723,7 +712,6 @@ func TestRespondToTaskV2(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestBatchesStateWriter(t *testing.T) { @@ -752,14 +740,11 @@ func TestBatchesStateWriter(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) // Directly panics -> Need to catch in retry. - /* - _, err = avsWriter.BatchesStateRetryable(bytes) - assert.NotNil(t, err) - fmt.Printf(": %s\n", err) - */ + _, err = avsWriter.BatchesStateRetryable(bytes) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) // Start anvil cmd, _, err = SetupAnvil(8545) @@ -776,7 +761,6 @@ func TestBatchesStateWriter(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestBalanceAt(t *testing.T) { @@ -802,7 +786,6 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(16)) assert.NotNil(t, err) @@ -823,7 +806,6 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } func TestBatchersBalances(t *testing.T) { @@ -849,13 +831,10 @@ func TestBatchersBalances(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) - /* - _, err = avsWriter.BatcherBalancesRetryable(sender_address) - assert.NotNil(t, err) - fmt.Printf(": %s\n", err) - */ + _, err = avsWriter.BatcherBalancesRetryable(sender_address) + assert.NotNil(t, err) + fmt.Printf(": %s\n", err) // Start anvil cmd, _, err = SetupAnvil(8545) @@ -872,5 +851,4 @@ func TestBatchersBalances(t *testing.T) { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(750 * time.Millisecond) } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index e6291487d..982ab3be7 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -2,6 +2,8 @@ package utils import ( "context" + "fmt" + "strings" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" eigentypes "github.com/Layr-Labs/eigensdk-go/types" @@ -10,10 +12,27 @@ import ( connection "github.com/yetanotherco/aligned_layer/core" ) -// Does not +/* +Errors: +- "not found": (Transient) Call successfully returns but the tx receipt was not found. +- "connect: connection refused": (Transient) Could not connect. +*/ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { // For if no receipt and no error TransactionReceipt return "not found" as an error catch all ref: https://github.com/ethereum/go-ethereum/blob/master/ethclient/ethclient.go#L313 - receipt_func := func() (*types.Receipt, error) { return client.TransactionReceipt(ctx, txHash) } + receipt_func := func() (*types.Receipt, error) { + tx, err := client.TransactionReceipt(ctx, txHash) + if err != nil { + // Note return type will be nil + if err.Error() == "not found" { + return nil, connection.TransientError{Inner: err} + } + if strings.Contains(err.Error(), "connect: connection refused") { + return nil, connection.TransientError{Inner: err} + } + return nil, connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return tx, err + } return connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } From 3258209a798420ceb177b3309401212efde77e18 Mon Sep 17 00:00:00 2001 From: nicolau Date: Mon, 4 Nov 2024 10:18:10 -0300 Subject: [PATCH 060/135] refactor: defined constants and better comments --- core/chainio/avs_writer.go | 15 +++++++++------ core/connection.go | 15 +++++++-------- core/utils/eth_client_utils.go | 10 +++++----- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index edcf90d38..b0f390cc1 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -21,8 +21,11 @@ import ( ) const ( - GasBumpPercentage int = 20 - // wait as much as 3 blocks time for the receipt + // How much to bump every retry (constant) + GasBaseBumpPercentage int = 20 + // An extra percentage to bump every retry i*5 (linear) + GasBumpIncrementalBumpPercentage int = 5 + // Wait as much as 3 blocks time for the receipt SendAggregateResponseReceiptTimeout time.Duration = time.Second * 36 ) @@ -82,9 +85,9 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E // it will try again bumping the last tx gas price based on `CalculateGasPriceBump` // This process happens indefinitely until the transaction is included. // -// Note: If the rpc endpoints fail, the retry will stop it returning a permanent error. -// This is because the retries are infinite and we want to prevent increasing the time between them too much as it is exponential. -// And we might also if the rpc is down for a good period of time, we might fall into an infinite waiting. +// Note: If the rpc endpoints fail, the retry mechanism stops, returning a permanent error. +// This is because retries are infinite and we want to prevent increasing the time between them too much as it is exponential. +// And, if the rpc is down for a good period of time, we might fall into an infinite waiting. func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, onRetry func()) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction @@ -124,7 +127,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, connection.PermanentError{Inner: err} } } - txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, GasBumpPercentage, i) + txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, GasBaseBumpPercentage, GasBumpIncrementalBumpPercentage, i) w.logger.Infof("Sending ResponseToTask transaction with a gas price of %v", txOpts.GasPrice) err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) diff --git a/core/connection.go b/core/connection.go index 0322d6888..d99f8a0a8 100644 --- a/core/connection.go +++ b/core/connection.go @@ -19,11 +19,13 @@ func (e PermanentError) Is(err error) bool { return ok } -// Same as Retry only that the functionToRetry can return a value upon correct execution +// Retries a given function in an exponential backoff manner and returns a value upon correct execution. +// It will retry calling the function while it returns a non permanent error, until the max retries. +// If maxTries == 0 then the retry function will run indefinitely until success or until a `PermanentError` is returned. func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { f := func() (*T, error) { val, err := functionToRetry() - if perm, ok := err.(PermanentError); err != nil && ok { + if perm, ok := err.(PermanentError); ok && err != nil { return nil, backoff.Permanent(perm.Inner) } return val, err @@ -46,15 +48,12 @@ func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, f } // Retries a given function in an exponential backoff manner. -// It will retry calling the function while it returns an error, until the max retries. -// If maxTries == 0 then the retry function will run indefinitely until success -// from the configuration are reached, or until a `PermanentError` is returned. -// The function to be retried should return `PermanentError` when the condition for stop retrying -// is met. +// It will retry calling the function while it returns a non permanent error, until the max retries. +// If maxTries == 0 then the retry function will run indefinitely. func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64) error { f := func() error { err := functionToRetry() - if perm, ok := err.(PermanentError); err != nil && ok { + if perm, ok := err.(PermanentError); ok && err != nil { return backoff.Permanent(perm.Inner) } return err diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index f81561993..b96b9140a 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -48,13 +48,13 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e } // Simple algorithm to calculate the gasPrice bump based on: -// the currentGasPrice, a base bump percentage, and the retry count. +// the currentGasPrice, a base bump percentage, a retry percentage, and the retry count. // Formula: currentGasPrice + (currentGasPrice * (baseBumpPercentage + retryCount * incrementalRetryPercentage) / 100) -func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercentage int, retryCount int) *big.Int { - // Incremental percentage increase for each retry attempt (5%) - incrementalRetryPercentage := new(big.Int).Mul(big.NewInt(5), big.NewInt(int64(retryCount))) +func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercentage int, retryAttemptPercentage int, retryCount int) *big.Int { + // Incremental percentage increase for each retry attempt (i*5%) + incrementalRetryPercentage := new(big.Int).Mul(big.NewInt(int64(retryAttemptPercentage)), big.NewInt(int64(retryCount))) - // Total bump percentage: base bump + incremental percentage based on retry count + // Total bump percentage: base bump + incremental retry percentage totalBumpPercentage := new(big.Int).Add(big.NewInt(int64(baseBumpPercentage)), incrementalRetryPercentage) // Calculate the bump amount: currentGasPrice * totalBumpPercentage / 100 From b1bc71f7ba5a95f6cd42e559dec9700086a7f914 Mon Sep 17 00:00:00 2001 From: nicolau Date: Mon, 4 Nov 2024 10:20:34 -0300 Subject: [PATCH 061/135] style: eol grafana .json --- grafana/provisioning/dashboards/aligned/aggregator_batcher.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index eb17d9f0b..d247b9910 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -1425,4 +1425,4 @@ "uid": "aggregator", "version": 15, "weekStart": "" -} \ No newline at end of file +} From 3614696f233f023735492ba15cbdc520714eaee0 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 5 Nov 2024 09:52:56 -0300 Subject: [PATCH 062/135] test: eth utils CalculateGasPriceBumpBasedOnRetry --- core/utils/eth_client_utils_test.go | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 core/utils/eth_client_utils_test.go diff --git a/core/utils/eth_client_utils_test.go b/core/utils/eth_client_utils_test.go new file mode 100644 index 000000000..739354b57 --- /dev/null +++ b/core/utils/eth_client_utils_test.go @@ -0,0 +1,38 @@ +package utils_test + +import ( + "math/big" + "testing" + + "github.com/yetanotherco/aligned_layer/core/utils" +) + +func TestCalculateGasPriceBumpBasedOnRetry(t *testing.T) { + n := 5 + baseBumpPercentage := 20 + incrementalRetryPercentage := 5 + gasPrices := [5]*big.Int{ + big.NewInt(3000000000), + big.NewInt(3000000000), + big.NewInt(4000000000), + big.NewInt(4000000000), + big.NewInt(5000000000)} + + expectedBumpedGasPrices := [5]*big.Int{ + big.NewInt(3600000000), + big.NewInt(3750000000), + big.NewInt(5200000000), + big.NewInt(5400000000), + big.NewInt(7000000000)} + + for i := range n { + currentGasPrice := gasPrices[i] + bumpedGasPrice := utils.CalculateGasPriceBumpBasedOnRetry(currentGasPrice, baseBumpPercentage, incrementalRetryPercentage, i) + expectedGasPrice := expectedBumpedGasPrices[i] + + if bumpedGasPrice.Cmp(expectedGasPrice) != 0 { + t.Errorf("Bumped gas price does not match expected gas price, expected value %v, got: %v", expectedGasPrice, bumpedGasPrice) + } + } + +} From bcf5456cbac43368babf64e283c574cfdd64ca98 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 5 Nov 2024 16:13:40 -0300 Subject: [PATCH 063/135] add transient errors + use struct capture in retry functions --- core/chainio/avs_subscriber.go | 213 ++++++++++++++++++++++++++------- core/chainio/avs_writer.go | 94 ++++++++++++--- core/connection.go | 6 +- core/utils/eth_client_utils.go | 3 + 4 files changed, 255 insertions(+), 61 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 71508bb81..08ae2a1f2 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "math/big" + "strings" "sync" "time" @@ -73,10 +74,12 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema // Retry() sub, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) + s.logger.Error("Failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries, "err", err) return nil, err } + s.logger.Info("Subscribed to new AlignedLayer tasks") + //TODO: collapse this // Retry() subFallback, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { @@ -148,10 +151,12 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema // Retry() sub, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) + s.logger.Error("Failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries, "err", err) return nil, err } + s.logger.Info("Subscribed to new AlignedLayer tasks") + //TODO: collapse this // Retry() subFallback, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { @@ -278,10 +283,10 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag var fromBlock uint64 - if *latestBlock < BlockInterval { + if latestBlock < BlockInterval { fromBlock = 0 } else { - fromBlock = *latestBlock - BlockInterval + fromBlock = latestBlock - BlockInterval } // Retry() @@ -330,10 +335,10 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag var fromBlock uint64 - if *latestBlock < BlockInterval { + if latestBlock < BlockInterval { fromBlock = 0 } else { - fromBlock = *latestBlock - BlockInterval + fromBlock = latestBlock - BlockInterval } // Retry() @@ -379,7 +384,7 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { return err } - if *currentBlock <= startBlock { // should really be == but just in case + if currentBlock <= startBlock { // should really be == but just in case // Subscribe to new head c := make(chan *types.Header) // Retry() @@ -390,7 +395,7 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // Read channel for the new block <-c - (*sub).Unsubscribe() + (sub).Unsubscribe() } return nil @@ -413,63 +418,156 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // |---RETRYABLE---| -func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (*uint64, error) { - latestBlock_func := func() (*uint64, error) { - latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) +func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { + var ( + latestBlock uint64 + err error + ) + latestBlock_func := func() (uint64, error) { + latestBlock, err = s.AvsContractBindings.ethClient.BlockNumber(ctx) if err != nil { latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) if err != nil { - return nil, err + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return latestBlock, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return latestBlock, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return latestBlock, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } } - return &latestBlock, err + return latestBlock, err } return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + var ( + logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator + err error + ) filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + if err != nil { + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return logs, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return logs, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return logs, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return logs, err } return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - + var ( + logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator + err error + ) filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + if err != nil { + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return logs, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return logs, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return logs, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return logs, err } return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (*struct { +func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int }, error) { - batchState_func := func() (*struct { + var ( + state struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + } + err error + ) + batchState_func := func() (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int }, error) { - state, err := s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) - return &state, err + state, err = s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) + if err != nil { + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return state, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return state, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return state, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return state, err } return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } -func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (*ethereum.Subscription, error) { - subscribeNewHead_func := func() (*ethereum.Subscription, error) { - sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) +func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { + var ( + sub ethereum.Subscription + err error + ) + subscribeNewHead_func := func() (ethereum.Subscription, error) { + sub, err = s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) if err != nil { sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) if err != nil { - return nil, err + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return sub, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } } - return &sub, err + return sub, err } return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } @@ -479,16 +577,30 @@ func SubscribeToNewTasksV2Retrayable( newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, logger sdklogging.Logger, ) (event.Subscription, error) { - subscribe_func := func() (*event.Subscription, error) { - sub, err := serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) - return &sub, err - } - sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + var ( + sub event.Subscription + err error + ) + subscribe_func := func() (event.Subscription, error) { + sub, err = serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) + if err != nil { + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return sub, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return sub, err } - logger.Info("Subscribed to new AlignedLayer tasks") - return *sub, nil + return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } func SubscribeToNewTasksV3Retryable( @@ -496,15 +608,28 @@ func SubscribeToNewTasksV3Retryable( newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, logger sdklogging.Logger, ) (event.Subscription, error) { - subscribe_func := func() (*event.Subscription, error) { - sub, err := serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) - return &sub, err - } - sub, err := connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - if err != nil { - // If the retries stop early we propogate the Permanent error to the caller. - return nil, fmt.Errorf("failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries) + var ( + sub event.Subscription + err error + ) + subscribe_func := func() (event.Subscription, error) { + sub, err = serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) + if err != nil { + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return sub, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return sub, err } - logger.Info("Subscribed to new AlignedLayer tasks") - return *sub, nil + return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 50fdefeff..7b5f02659 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "strings" "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients" @@ -178,13 +179,29 @@ func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byt // |---RETRYABLE---| func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { + var ( + tx *types.Transaction + err error + ) respondToTaskV2_func := func() (*types.Transaction, error) { - tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { // Retry with fallback tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { - return nil, err + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return tx, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return tx, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return tx, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } } return tx, err @@ -197,33 +214,68 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { Responded bool RespondToTaskFeeLimit *big.Int }, error) { - batchesState_func := func() (*struct { + var ( + state struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + } + err error + ) + batchesState_func := func() (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int }, error) { - state, err := w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) + state, err = w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) if err != nil { // Retry with fallback state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) // If error is not nil throw out result in state! if err != nil { - return &state, err + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return state, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return state, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return state, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } } - return &state, err + return state, err } - state, err := connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) - return *state, err + return connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) } func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { + var ( + batcherBalance *big.Int + err error + ) batcherBalances_func := func() (*big.Int, error) { - batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + batcherBalance, err = w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) if err != nil { batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) if err != nil { - return nil, err + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return batcherBalance, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return batcherBalance, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return batcherBalance, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } } return batcherBalance, err @@ -232,16 +284,30 @@ func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big } func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { - + var ( + aggregatorBalance *big.Int + err error + ) balanceAt_func := func() (*big.Int, error) { - aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) + aggregatorBalance, err = w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) //aggregatorBalance, err := connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) //aggregatorBalance, err = connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { - // Ignore and continue. - return nil, err + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return aggregatorBalance, err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return aggregatorBalance, err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return aggregatorBalance, connection.TransientError{Inner: err} + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } } return aggregatorBalance, err diff --git a/core/connection.go b/core/connection.go index 7a21908e4..9d76f2669 100644 --- a/core/connection.go +++ b/core/connection.go @@ -38,11 +38,11 @@ const RetryFactor = 2 const NumRetries = 3 // Same as Retry only that the functionToRetry can return a value upon correct execution -func RetryWithData[T any](functionToRetry func() (*T, error), minDelay uint64, factor float64, maxTries uint64) (*T, error) { +func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, factor float64, maxTries uint64) (T, error) { i := 0 - f := func() (*T, error) { + f := func() (T, error) { var ( - val *T + val T err error ) func() { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 982ab3be7..4187d0ba5 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -29,6 +29,9 @@ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx conte if strings.Contains(err.Error(), "connect: connection refused") { return nil, connection.TransientError{Inner: err} } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return nil, connection.TransientError{Inner: err} + } return nil, connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } return tx, err From b1b5769c217670f3474707b435473473f8572258 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 5 Nov 2024 16:13:44 -0300 Subject: [PATCH 064/135] add tests --- core/connection_test.go | 178 ++++++++++++++++++++++++++++------------ 1 file changed, 127 insertions(+), 51 deletions(-) diff --git a/core/connection_test.go b/core/connection_test.go index ad1d5ca68..a87649752 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -161,27 +161,28 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { // Assert Call succeeds when Anvil running _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) - if _, ok := err.(*connection.TransientError); !ok { - t.Errorf("Retry on failure did not return backoff Permanent Error: %s", err) - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) - - } if err.Error() != "not found" { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) return } // Kill Anvil - if err := cmd.Process.Kill(); err != nil { + if err = cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return } - // Errors out but "not found" - receipt, err := utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) - fmt.Printf("error fetching from killed anvil: %v\n", err) - assert.Nil(t, receipt, "Receipt not empty") - assert.NotEqual(t, err.Error(), "not found") + _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + assert.NotNil(t, err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, client, err = SetupAnvil(8545) @@ -190,9 +191,10 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { } _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) - assert.NotNil(t, err, "Call to Anvil failed") + assert.NotNil(t, err) if err.Error() != "not found" { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) + return } // Kill Anvil at end of test @@ -329,10 +331,8 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) } - fmt.Printf("Subscribing") _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) assert.Nil(t, err) - // TODO: Find exact error to assert // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -342,7 +342,15 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -352,7 +360,6 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -381,7 +388,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.Nil(t, err) - // TODO: Find exact error to assert // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -391,7 +397,15 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -401,7 +415,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -424,7 +437,7 @@ func TestBlockNumber(t *testing.T) { return } _, err = sub.BlockNumberRetryable(context.Background()) - assert.Nil(t, err, "Failed to Retrieve Block Number") + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -434,7 +447,15 @@ func TestBlockNumber(t *testing.T) { _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -444,7 +465,6 @@ func TestBlockNumber(t *testing.T) { _, err = sub.BlockNumberRetryable(context.Background()) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -466,8 +486,7 @@ func TestFilterBatchV2(t *testing.T) { return } _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) - assert.Nil(t, err, "Failed to Retrieve Block Number") - // TODO: Find exact error to assert + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -477,7 +496,15 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -487,7 +514,6 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -509,8 +535,7 @@ func TestFilterBatchV3(t *testing.T) { return } _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) - //TODO: Find error to assert - assert.Nil(t, err, "Succeeded in filtering logs") + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -520,7 +545,15 @@ func TestFilterBatchV3(t *testing.T) { _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -530,7 +563,6 @@ func TestFilterBatchV3(t *testing.T) { _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -555,7 +587,7 @@ func TestBatchesStateSubscriber(t *testing.T) { zero_bytes := [32]byte{} _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) //TODO: Find exact failure error - assert.Nil(t, err, "BatchesState") + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -565,7 +597,15 @@ func TestBatchesStateSubscriber(t *testing.T) { _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -575,7 +615,6 @@ func TestBatchesStateSubscriber(t *testing.T) { _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -599,7 +638,7 @@ func TestSubscribeNewHead(t *testing.T) { } avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) - assert.Nil(t, err, "Should be 0") + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -609,7 +648,15 @@ func TestSubscribeNewHead(t *testing.T) { _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -619,7 +666,6 @@ func TestSubscribeNewHead(t *testing.T) { _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -676,7 +722,7 @@ func TestRespondToTaskV2(t *testing.T) { // Panics if 0 object passed _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) - assert.NotNil(t, err, "Should error") + assert.NotNil(t, err) // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" if !strings.Contains(err.Error(), "execution reverted: custom error 0x2396d34e:") { t.Errorf("Respond to task V2 Retryable did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") @@ -691,7 +737,15 @@ func TestRespondToTaskV2(t *testing.T) { _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("RespondToTaksV2 Emitted non-Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("RespondToTaskV2 did not return expected error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -699,8 +753,9 @@ func TestRespondToTaskV2(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } + //TODO: address custom error _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) - assert.NotNil(t, err, "Should error") + assert.NotNil(t, err) // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" if !strings.Contains(err.Error(), "execution reverted: custom error 0x2396d34e:") { t.Errorf("Respond to task V2 Retryable did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") @@ -733,7 +788,7 @@ func TestBatchesStateWriter(t *testing.T) { num.FillBytes(bytes[:]) _, err = avsWriter.BatchesStateRetryable(bytes) - assert.Nil(t, err, "Should be 0") + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -744,7 +799,15 @@ func TestBatchesStateWriter(t *testing.T) { // Directly panics -> Need to catch in retry. _, err = avsWriter.BatchesStateRetryable(bytes) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("BatchesStateWriter Emitted non-Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("BatchesStateWriter did not contain expected error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -754,7 +817,6 @@ func TestBatchesStateWriter(t *testing.T) { _, err = avsWriter.BatchesStateRetryable(bytes) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -778,8 +840,8 @@ func TestBalanceAt(t *testing.T) { //TODO: Source Aggregator Address aggregator_address := common.HexToAddress("0x0") - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(16)) - assert.Nil(t, err, "Should be 0") + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(14)) + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -787,9 +849,17 @@ func TestBalanceAt(t *testing.T) { return } - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(16)) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(14)) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("BalanceAt Emitted non-Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("BalanceAt did not return expected error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -797,9 +867,8 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(16)) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(14)) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -824,7 +893,7 @@ func TestBatchersBalances(t *testing.T) { sender_address := common.HexToAddress("0x0") _, err = avsWriter.BatcherBalancesRetryable(sender_address) - assert.Nil(t, err, "Should be 0") + assert.Nil(t, err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { @@ -834,7 +903,15 @@ func TestBatchersBalances(t *testing.T) { _, err = avsWriter.BatcherBalancesRetryable(sender_address) assert.NotNil(t, err) - fmt.Printf(": %s\n", err) + // Assert returned error is both transient error and contains the expected error msg. + if _, ok := err.(connection.TransientError); !ok { + fmt.Printf("BatchersBalances Emitted non-Transient error: %s\n", err) + return + } + if !strings.Contains(err.Error(), "connect: connection refused") { + fmt.Printf("BatchersBalances did not return expected error: %s\n", err) + return + } // Start anvil cmd, _, err = SetupAnvil(8545) @@ -844,7 +921,6 @@ func TestBatchersBalances(t *testing.T) { _, err = avsWriter.BatcherBalancesRetryable(sender_address) assert.Nil(t, err) - fmt.Printf("Error Failed to Retrieve Block Number: %s\n", err) // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { From b706fbf591f8a85f3d47947134ef1cf8006fb875 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 5 Nov 2024 16:41:50 -0300 Subject: [PATCH 065/135] rm unneeded cmts --- aggregator/pkg/aggregator.go | 5 ----- core/chainio/avs_subscriber.go | 32 ++++++++------------------------ core/chainio/avs_writer.go | 12 ------------ core/connection_test.go | 6 ++---- 4 files changed, 10 insertions(+), 45 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 9907c3251..b4f4d708e 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -288,8 +288,6 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.telemetry.LogTaskError(batchData.BatchMerkleRoot, err) } -// TODO(pat): should we be retrying the entire Send -> Receive Receipt step? For bumping the gas fee this is needed. - // / Sends response to contract and waits for transaction receipt // / Returns error if it fails to send tx or receipt is not found func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*gethtypes.Receipt, error) { @@ -300,7 +298,6 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "senderAddress", hex.EncodeToString(senderAddress[:]), "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) - // Retry txHash, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { agg.walletMutex.Unlock() @@ -312,7 +309,6 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Sending aggregated response for batch %s", hex.EncodeToString(batchIdentifierHash[:])) - // Retry receipt, err := utils.WaitForTransactionReceiptRetryable( agg.AggregatorConfig.BaseConfig.EthRpcClient, context.Background(), *txHash) if err != nil { @@ -367,7 +363,6 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(QUORUM_NUMBER)} quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(QUORUM_THRESHOLD)} - // Retry err := agg.InitializeNewTaskRetryable(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) // FIXME(marian): When this errors, should we retry initializing new task? Logging fatal for now. if err != nil { diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 08ae2a1f2..cf051a95e 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -71,21 +71,20 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) // Subscribe to new tasks - // Retry() + // NOTE: Should we consolidate these subscribers behind two interfaces sub, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries, "err", err) + s.logger.Error("Failed to subscribe to new AlignedLayer tasks after %d retries", connection.NumRetries, "err", err) return nil, err } - s.logger.Info("Subscribed to new AlignedLayer tasks") + s.logger.Info("Subscribed to new AlignedLayer V2 tasks") - //TODO: collapse this - // Retry() subFallback, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) + s.logger.Error("Failed to subscribe to new AlignedLayer tasks %d retries", connection.NumRetries, "err", err) return nil, err } + s.logger.Info("Subscribed to new AlignedLayer V2 tasks") // create a new channel to foward errors errorChannel := make(chan error) @@ -102,7 +101,6 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case newBatch := <-internalChannel: s.processNewBatchV2(newBatch, batchesSet, newBatchMutex, newTaskCreatedChan) case <-pollLatestBatchTicker.C: - //TODO: Retry() latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV2() if err != nil { s.logger.Debug("Failed to get latest task from blockchain", "err", err) @@ -123,7 +121,6 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - // Retry() sub, err = SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -131,7 +128,6 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - // Retry() subFallback, err = SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -148,21 +144,20 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) // Subscribe to new tasks - // Retry() sub, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { s.logger.Error("Failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries, "err", err) return nil, err } - s.logger.Info("Subscribed to new AlignedLayer tasks") + s.logger.Info("Subscribed to new AlignedLayer V3 tasks") //TODO: collapse this - // Retry() subFallback, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks", "err", err) + s.logger.Error("Failed to subscribe to new AlignedLayer %d tasks", MaxRetries, "err", err) return nil, err } + s.logger.Info("Subscribed to new AlignedLayer V3 tasks") // create a new channel to foward errors errorChannel := make(chan error) @@ -179,7 +174,6 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case newBatch := <-internalChannel: s.processNewBatchV3(newBatch, batchesSet, newBatchMutex, newTaskCreatedChan) case <-pollLatestBatchTicker.C: - //TODO: Retry() latestBatch, err := s.getLatestNotRespondedTaskFromEthereumV3() if err != nil { s.logger.Debug("Failed to get latest task from blockchain", "err", err) @@ -200,7 +194,6 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - // Retry() sub, err = SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -208,7 +201,6 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - // Retry() subFallback, err = SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { errorChannel <- err @@ -275,7 +267,6 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { - // Retry() latestBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return nil, err @@ -289,7 +280,6 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag fromBlock = latestBlock - BlockInterval } - // Retry() logs, err := s.FilterBatchV2Retryable(fromBlock, context.Background()) if err != nil { return nil, err @@ -312,7 +302,6 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - // Retry() state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err @@ -327,7 +316,6 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { - // Retry() latestBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return nil, err @@ -341,7 +329,6 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag fromBlock = latestBlock - BlockInterval } - // Retry() logs, err := s.FilterBatchV3Retryable(fromBlock, context.Background()) if err != nil { return nil, err @@ -364,7 +351,6 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - // Retry() state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err @@ -378,7 +364,6 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag } func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { - // Retry() currentBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return err @@ -387,7 +372,6 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { if currentBlock <= startBlock { // should really be == but just in case // Subscribe to new head c := make(chan *types.Header) - // Retry() sub, err := s.SubscribeNewHeadRetryable(context.Background(), c) if err != nil { return err diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 7b5f02659..6787eb3f1 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -75,7 +75,6 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*common.Hash, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction - // Retry tx, err := w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err @@ -89,7 +88,6 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // Send the transaction txOpts.NoSend = false txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit - // Retry tx, err = w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err @@ -106,7 +104,6 @@ func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bin w.logger.Info("Simulated cost", "cost", simulatedCost) // Get RespondToTaskFeeLimit - // Retry batchState, err := w.BatchesStateRetryable(batchIdentifierHash) if err != nil { // Fallback also failed @@ -124,16 +121,13 @@ func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bin return fmt.Errorf("cost of transaction is higher than Batch.RespondToTaskFeeLimit") } - // Retry return w.compareBalances(respondToTaskFeeLimit, aggregatorAddress, senderAddress) } func (w *AvsWriter) compareBalances(amount *big.Int, aggregatorAddress common.Address, senderAddress [20]byte) error { - // Retry if err := w.compareAggregatorBalance(amount, aggregatorAddress); err != nil { return err } - // Retry if err := w.compareBatcherBalance(amount, senderAddress); err != nil { return err } @@ -144,8 +138,6 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - // TODO(pat): Separate Retry from call - // Retry aggregatorBalance, err := w.BalanceAtRetryable(ctx, aggregatorAddress, nil) if err != nil { // Ignore and continue. @@ -161,8 +153,6 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byte) error { // Get batcher balance - // TODO(pat): Separate Retry from call - // Retry batcherBalance, err := w.BatcherBalancesRetryable(senderAddress) if err != nil { // Ignore and continue. @@ -186,7 +176,6 @@ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl respondToTaskV2_func := func() (*types.Transaction, error) { tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { - // Retry with fallback tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { // Note return type will be nil @@ -229,7 +218,6 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { }, error) { state, err = w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) if err != nil { - // Retry with fallback state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) // If error is not nil throw out result in state! if err != nil { diff --git a/core/connection_test.go b/core/connection_test.go index a87649752..250e3b6a5 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -720,7 +720,7 @@ func TestRespondToTaskV2(t *testing.T) { aggregator_address := common.HexToAddress("0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690") zero_bytes := [32]byte{} - // Panics if 0 object passed + // NOTE: With zero bytes the tx reverts _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" @@ -753,7 +753,7 @@ func TestRespondToTaskV2(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - //TODO: address custom error + // NOTE: With zero bytes the tx reverts _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" @@ -796,7 +796,6 @@ func TestBatchesStateWriter(t *testing.T) { return } - // Directly panics -> Need to catch in retry. _, err = avsWriter.BatchesStateRetryable(bytes) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. @@ -889,7 +888,6 @@ func TestBatchersBalances(t *testing.T) { if err != nil { return } - //TODO: Source real one sender_address := common.HexToAddress("0x0") _, err = avsWriter.BatcherBalancesRetryable(sender_address) From 280e6c41c3e619e09dd9b23deb91caccd75542d5 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 5 Nov 2024 16:42:01 -0300 Subject: [PATCH 066/135] refactor ProcessNewSignature --- aggregator/pkg/server.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 51ae435d9..9f232df0f 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -3,8 +3,10 @@ package pkg import ( "context" "encoding/hex" + "fmt" "net/http" "net/rpc" + "strings" "time" "github.com/Layr-Labs/eigensdk-go/crypto/bls" @@ -86,7 +88,6 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t agg.logger.Info("Starting bls signature process") go func() { - // Retry err := agg.ProcessNewSignatureRetryable( context.Background(), taskIndex, signedTaskResponse.BatchIdentifierHash, &signedTaskResponse.BlsSignature, signedTaskResponse.OperatorId, @@ -131,11 +132,29 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // Error throw is ______ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { + var err error processNewSignature_func := func() error { - return agg.blsAggregationService.ProcessNewSignature( + err = agg.blsAggregationService.ProcessNewSignature( ctx, taskIndex, taskResponse, blsSignature, operatorId, ) + if err != nil { + // Note return type will be nil + if err.Error() == "not found" { + err = connection.TransientError{Inner: err} + return err + } + if strings.Contains(err.Error(), "connect: connection refused") { + err = connection.TransientError{Inner: err} + return err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + err = connection.TransientError{Inner: err} + return err + } + err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return err } return connection.Retry(processNewSignature_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) From 3d3d243101bb6c9c27e888cc55000f50130fe176 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 5 Nov 2024 19:07:43 -0300 Subject: [PATCH 067/135] ci: fix --- core/utils/eth_client_utils_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/utils/eth_client_utils_test.go b/core/utils/eth_client_utils_test.go index 739354b57..72d718ed2 100644 --- a/core/utils/eth_client_utils_test.go +++ b/core/utils/eth_client_utils_test.go @@ -8,7 +8,6 @@ import ( ) func TestCalculateGasPriceBumpBasedOnRetry(t *testing.T) { - n := 5 baseBumpPercentage := 20 incrementalRetryPercentage := 5 gasPrices := [5]*big.Int{ @@ -25,7 +24,7 @@ func TestCalculateGasPriceBumpBasedOnRetry(t *testing.T) { big.NewInt(5400000000), big.NewInt(7000000000)} - for i := range n { + for i := 0; i < len(gasPrices); i++ { currentGasPrice := gasPrices[i] bumpedGasPrice := utils.CalculateGasPriceBumpBasedOnRetry(currentGasPrice, baseBumpPercentage, incrementalRetryPercentage, i) expectedGasPrice := expectedBumpedGasPrices[i] From 251cd39a805947243d6beb4ad906e083f927a721 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 5 Nov 2024 23:02:41 -0300 Subject: [PATCH 068/135] cmt in server --- aggregator/pkg/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 9f232df0f..f876593f7 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -57,7 +57,7 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t taskIndex := uint32(0) ok := false - //TODO(pat): This is retried but waits are internal map aka is not a fallable connection + // NOTE: Since this does not interact with a fallible connection waiting we use a different retry mechanism than the rest of the aggregator. for i := 0; i < waitForEventRetries; i++ { agg.taskMutex.Lock() agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Starting processing of Response") From aa3919919821fe9299dd9f25b0dd7add98c9b946 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Tue, 5 Nov 2024 23:08:11 -0300 Subject: [PATCH 069/135] lint --- core/connection_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/connection_test.go b/core/connection_test.go index 250e3b6a5..b0e164c4b 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -637,7 +637,7 @@ func TestSubscribeNewHead(t *testing.T) { return } - avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) + _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) // Kill Anvil at end of test From 9fa785c48530a3c1df88a5b8a8d2b63a7dda0ac0 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 12:45:16 -0300 Subject: [PATCH 070/135] fix merge change --- aggregator/pkg/aggregator.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 28417c92f..e205a2c87 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -272,19 +272,14 @@ func (agg *Aggregator) handleBlsAggServiceResponse(blsAggServiceResp blsagg.BlsA agg.logger.Info("Sending aggregated response onchain", "taskIndex", blsAggServiceResp.TaskIndex, "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - for i := 0; i < MaxSentTxRetries; i++ { - receipt, err := agg.sendAggregatedResponse(batchIdentifierHash, batchData.BatchMerkleRoot, batchData.SenderAddress, nonSignerStakesAndSignature) - if err == nil { - agg.telemetry.TaskSentToEthereum(batchData.BatchMerkleRoot, receipt.TxHash.String()) - agg.logger.Info("Aggregator successfully responded to task", - "taskIndex", blsAggServiceResp.TaskIndex, - "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - - return - } + receipt, err := agg.sendAggregatedResponse(batchIdentifierHash, batchData.BatchMerkleRoot, batchData.SenderAddress, nonSignerStakesAndSignature) + if err == nil { + agg.telemetry.TaskSentToEthereum(batchData.BatchMerkleRoot, receipt.TxHash.String()) + agg.logger.Info("Aggregator successfully responded to task", + "taskIndex", blsAggServiceResp.TaskIndex, + "batchIdentifierHash", "0x"+hex.EncodeToString(batchIdentifierHash[:])) - // Sleep for a bit before retrying - time.Sleep(2 * time.Second) + return } agg.logger.Error("Aggregator failed to respond to task, this batch will be lost", From 2392c208bd6586904dc2bd616ed382b73bcf5347 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 12:56:10 -0300 Subject: [PATCH 071/135] add attribution comment --- core/connection.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/connection.go b/core/connection.go index 9d76f2669..2067f6b44 100644 --- a/core/connection.go +++ b/core/connection.go @@ -7,6 +7,11 @@ import ( "github.com/cenkalti/backoff/v4" ) +/* +This retry library was inspired by and uses Cenk Alti (https://github.com/cenkalti) backoff library (https://github.com/cenkalti/backoff). +We would like to thank him for his great work. +*/ + type PermanentError struct { Inner error } From 2d5cc863ea1e4e5f37446f57d2852ea9effe5d49 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 14:44:05 -0300 Subject: [PATCH 072/135] add maxInterval as option --- core/connection.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/connection.go b/core/connection.go index 2067f6b44..8689be6fb 100644 --- a/core/connection.go +++ b/core/connection.go @@ -41,9 +41,10 @@ func (e TransientError) Is(err error) bool { const MinDelay = 1000 const RetryFactor = 2 const NumRetries = 3 +const MaxInterval = 60000 // Same as Retry only that the functionToRetry can return a value upon correct execution -func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, factor float64, maxTries uint64) (T, error) { +func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, factor float64, maxTries uint64, maxInterval uint64) (T, error) { i := 0 f := func() (T, error) { var ( @@ -73,7 +74,8 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) - expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) + maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(MaxInterval)) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption) var maxRetriesBackoff backoff.BackOff if maxTries > 0 { From 035f19a819d6fab65105310b5b4ab7faf54bd372 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 15:00:56 -0300 Subject: [PATCH 073/135] add maxInterval option --- aggregator/pkg/aggregator.go | 2 +- aggregator/pkg/server.go | 2 +- core/chainio/avs_subscriber.go | 14 +++++++------- core/chainio/avs_writer.go | 8 ++++---- core/connection.go | 7 ++++--- core/connection_test.go | 4 ++-- core/utils/eth_client_utils.go | 2 +- 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index e205a2c87..4c474da64 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -390,7 +390,7 @@ func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreated // Returns an error if the return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) } - return connection.Retry(initilizeNewTask_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.Retry(initilizeNewTask_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } // Long-lived goroutine that periodically checks and removes old Tasks from stored Maps diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index f876593f7..12c401d9d 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -157,5 +157,5 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd return err } - return connection.Retry(processNewSignature_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.Retry(processNewSignature_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index cf051a95e..679b872a6 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -428,7 +428,7 @@ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error } return latestBlock, err } - return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { @@ -456,7 +456,7 @@ func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Con } return logs, err } - return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { @@ -483,7 +483,7 @@ func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Con } return logs, err } - return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { @@ -523,7 +523,7 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte return state, err } - return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { @@ -553,7 +553,7 @@ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- } return sub, err } - return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func SubscribeToNewTasksV2Retrayable( @@ -584,7 +584,7 @@ func SubscribeToNewTasksV2Retrayable( } return sub, err } - return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func SubscribeToNewTasksV3Retryable( @@ -615,5 +615,5 @@ func SubscribeToNewTasksV3Retryable( } return sub, err } - return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 6787eb3f1..cc85c84c6 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -195,7 +195,7 @@ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl } return tx, err } - return connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { @@ -238,7 +238,7 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { } return state, err } - return connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { @@ -268,7 +268,7 @@ func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big } return batcherBalance, err } - return connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { @@ -300,5 +300,5 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co } return aggregatorBalance, err } - return connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } diff --git a/core/connection.go b/core/connection.go index 8689be6fb..dcd23800f 100644 --- a/core/connection.go +++ b/core/connection.go @@ -74,7 +74,7 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) - maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(MaxInterval)) + maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(maxInterval)) expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption) var maxRetriesBackoff backoff.BackOff @@ -93,7 +93,7 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa // from the configuration are reached, or until a `PermanentError` is returned. // The function to be retried should return `PermanentError` when the condition for stop retrying // is met. -func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64) error { +func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64, maxInterval uint64) error { i := 0 f := func() error { err := functionToRetry() @@ -108,7 +108,8 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) - expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption) + maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(maxInterval)) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption) var maxRetriesBackoff backoff.BackOff if maxTries > 0 { diff --git a/core/connection_test.go b/core/connection_test.go index b0e164c4b..a2a5d03ea 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -41,7 +41,7 @@ func TestRetryWithData(t *testing.T) { x, err := DummyFunction(43) return &x, err } - data, err := connection.RetryWithData(function, 1000, 2, 3) + data, err := connection.RetryWithData(function, 1000, 2, 3, connection.MaxInterval) if err != nil { t.Errorf("Retry error!: %s", err) } else { @@ -54,7 +54,7 @@ func TestRetry(t *testing.T) { _, err := DummyFunction(43) return err } - err := connection.Retry(function, 1000, 2, 3) + err := connection.Retry(function, 1000, 2, 3, connection.MaxInterval) if err != nil { t.Errorf("Retry error!: %s", err) } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 4187d0ba5..9407c6745 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -36,7 +36,7 @@ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx conte } return tx, err } - return connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) + return connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) } func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { From 97fac6a1a89a71132b3af700f689d2c78b341f08 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 15:19:33 -0300 Subject: [PATCH 074/135] remove unused i variable --- core/connection.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/connection.go b/core/connection.go index dcd23800f..287157d47 100644 --- a/core/connection.go +++ b/core/connection.go @@ -45,7 +45,6 @@ const MaxInterval = 60000 // Same as Retry only that the functionToRetry can return a value upon correct execution func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, factor float64, maxTries uint64, maxInterval uint64) (T, error) { - i := 0 f := func() (T, error) { var ( val T @@ -62,7 +61,6 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa } }() val, err = functionToRetry() - i++ if perm, ok := err.(PermanentError); err != nil && ok { err = backoff.Permanent(perm.Inner) } @@ -94,10 +92,8 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa // The function to be retried should return `PermanentError` when the condition for stop retrying // is met. func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64, maxInterval uint64) error { - i := 0 f := func() error { err := functionToRetry() - i++ if perm, ok := err.(PermanentError); err != nil && ok { return backoff.Permanent(perm.Inner) } From 2bf4a4816c815626ce7073becfccf520910f26a9 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 15:27:43 -0300 Subject: [PATCH 075/135] refactor to add permanent backoff conversion function + add panic catch to Retry() --- core/connection.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/core/connection.go b/core/connection.go index 287157d47..da83dd6dc 100644 --- a/core/connection.go +++ b/core/connection.go @@ -38,6 +38,13 @@ func (e TransientError) Is(err error) bool { return ok } +func convertToBackOffPermanentError(err error) error { + if perm, ok := err.(PermanentError); err != nil && ok { + return backoff.Permanent(perm.Inner) + } + return err +} + const MinDelay = 1000 const RetryFactor = 2 const NumRetries = 3 @@ -61,9 +68,7 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa } }() val, err = functionToRetry() - if perm, ok := err.(PermanentError); err != nil && ok { - err = backoff.Permanent(perm.Inner) - } + err = convertToBackOffPermanentError(err) }() return val, err } @@ -93,10 +98,20 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa // is met. func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64, maxInterval uint64) error { f := func() error { - err := functionToRetry() - if perm, ok := err.(PermanentError); err != nil && ok { - return backoff.Permanent(perm.Inner) - } + var err error + func() { + defer func() { + if r := recover(); r != nil { + if panic_err, ok := r.(error); ok { + err = TransientError{panic_err} + } else { + err = TransientError{fmt.Errorf("panicked: %v", panic_err)} + } + } + }() + err = functionToRetry() + err = convertToBackOffPermanentError(err) + }() return err } From 801f9ed4f8e2809b0f7f6552adda83d62bc139f1 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 15:35:26 -0300 Subject: [PATCH 076/135] rm unneeded comments --- core/chainio/avs_writer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index cc85c84c6..4b6d4aec3 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -278,10 +278,8 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co ) balanceAt_func := func() (*big.Int, error) { aggregatorBalance, err = w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) - //aggregatorBalance, err := connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) - //aggregatorBalance, err = connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries) if err != nil { // Note return type will be nil if err.Error() == "not found" { From ca2da624e2e4c07a77795de9a9832a728a09402f Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 15:53:42 -0300 Subject: [PATCH 077/135] fix parameters for BalanceAt test --- core/connection.go | 15 ++++++--------- core/connection_test.go | 7 ++++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/connection.go b/core/connection.go index da83dd6dc..3eb864b1c 100644 --- a/core/connection.go +++ b/core/connection.go @@ -38,13 +38,6 @@ func (e TransientError) Is(err error) bool { return ok } -func convertToBackOffPermanentError(err error) error { - if perm, ok := err.(PermanentError); err != nil && ok { - return backoff.Permanent(perm.Inner) - } - return err -} - const MinDelay = 1000 const RetryFactor = 2 const NumRetries = 3 @@ -68,7 +61,9 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa } }() val, err = functionToRetry() - err = convertToBackOffPermanentError(err) + if perm, ok := err.(PermanentError); err != nil && ok { + err = backoff.Permanent(perm.Inner) + } }() return val, err } @@ -110,7 +105,9 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri } }() err = functionToRetry() - err = convertToBackOffPermanentError(err) + if perm, ok := err.(PermanentError); err != nil && ok { + err = backoff.Permanent(perm.Inner) + } }() return err } diff --git a/core/connection_test.go b/core/connection_test.go index a2a5d03ea..360c6cb49 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -838,8 +838,9 @@ func TestBalanceAt(t *testing.T) { } //TODO: Source Aggregator Address aggregator_address := common.HexToAddress("0x0") + blockHeight := big.NewInt(13) - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(14)) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) // Kill Anvil at end of test @@ -848,7 +849,7 @@ func TestBalanceAt(t *testing.T) { return } - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(14)) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(connection.TransientError); !ok { @@ -866,7 +867,7 @@ func TestBalanceAt(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, big.NewInt(14)) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) // Kill Anvil at end of test From 7aff569659db266bbc2d0747f77d629e617d077b Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 15:56:46 -0300 Subject: [PATCH 078/135] add comment --- core/connection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/connection.go b/core/connection.go index 3eb864b1c..e0008aaee 100644 --- a/core/connection.go +++ b/core/connection.go @@ -61,6 +61,7 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa } }() val, err = functionToRetry() + // Convert the returned `PermanentError` (our implementation) to `backoff.PermanentError`. if perm, ok := err.(PermanentError); err != nil && ok { err = backoff.Permanent(perm.Inner) } @@ -105,6 +106,7 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri } }() err = functionToRetry() + // Convert the returned `PermanentError`` (our implementation) to a `backoff.PermanentError`` if perm, ok := err.(PermanentError); err != nil && ok { err = backoff.Permanent(perm.Inner) } From 64705f34563d15fcc4db3f19e0156fa34bbec8e6 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 15:58:31 -0300 Subject: [PATCH 079/135] nits --- core/connection.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/connection.go b/core/connection.go index e0008aaee..a0c37de09 100644 --- a/core/connection.go +++ b/core/connection.go @@ -62,6 +62,7 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa }() val, err = functionToRetry() // Convert the returned `PermanentError` (our implementation) to `backoff.PermanentError`. + //This exits the retry loop in the `backoff` library. if perm, ok := err.(PermanentError); err != nil && ok { err = backoff.Permanent(perm.Inner) } @@ -106,7 +107,8 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri } }() err = functionToRetry() - // Convert the returned `PermanentError`` (our implementation) to a `backoff.PermanentError`` + // Convert the returned `PermanentError` (our implementation) to a `backoff.PermanentError`. + //This exits the retry loop in the `backoff` library. if perm, ok := err.(PermanentError); err != nil && ok { err = backoff.Permanent(perm.Inner) } From 0b338d7ae8470bcfad0967bd4e818e7b1e149df8 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 16:43:47 -0300 Subject: [PATCH 080/135] remove logs --- core/connection_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/connection_test.go b/core/connection_test.go index 360c6cb49..3c909bc85 100644 --- a/core/connection_test.go +++ b/core/connection_test.go @@ -27,7 +27,6 @@ import ( ) func DummyFunction(x uint64) (uint64, error) { - fmt.Println("Starting Anvil on Port ") if x == 42 { return 0, connection.PermanentError{Inner: fmt.Errorf("Permanent error!")} } else if x < 42 { @@ -41,11 +40,9 @@ func TestRetryWithData(t *testing.T) { x, err := DummyFunction(43) return &x, err } - data, err := connection.RetryWithData(function, 1000, 2, 3, connection.MaxInterval) + _, err := connection.RetryWithData(function, 1000, 2, 3, connection.MaxInterval) if err != nil { t.Errorf("Retry error!: %s", err) - } else { - fmt.Printf("DATA: %d\n", data) } } From 88a22152ec6a5054e0f80496260ef43e4d2c51f6 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Wed, 6 Nov 2024 17:46:54 -0300 Subject: [PATCH 081/135] rm "not found" from unneeded functions --- core/chainio/avs_subscriber.go | 29 ----------------------------- core/chainio/avs_writer.go | 16 ---------------- 2 files changed, 45 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 679b872a6..5913e502e 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -412,10 +412,6 @@ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error if err != nil { latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) if err != nil { - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return latestBlock, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return latestBlock, err @@ -441,10 +437,6 @@ func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Con logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return logs, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return logs, err @@ -468,10 +460,6 @@ func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Con logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return logs, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return logs, err @@ -507,10 +495,6 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte state, err = s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return state, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return state, err @@ -537,10 +521,6 @@ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return sub, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return sub, err @@ -569,10 +549,6 @@ func SubscribeToNewTasksV2Retrayable( sub, err = serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return sub, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return sub, err @@ -599,11 +575,6 @@ func SubscribeToNewTasksV3Retryable( subscribe_func := func() (event.Subscription, error) { sub, err = serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) if err != nil { - // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return sub, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return sub, err diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 4b6d4aec3..978c5009d 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -179,10 +179,6 @@ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return tx, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return tx, err @@ -222,10 +218,6 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { // If error is not nil throw out result in state! if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return state, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return state, err @@ -252,10 +244,6 @@ func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return batcherBalance, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return batcherBalance, err @@ -282,10 +270,6 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) if err != nil { // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return aggregatorBalance, err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return aggregatorBalance, err From 6ead6ed3cccedd1aa805fc04d968ca995068e9cf Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 11:54:45 -0300 Subject: [PATCH 082/135] rename connections to retry --- core/{connection.go => retry.go} | 2 +- core/{connection_test.go => retry_test.go} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename core/{connection.go => retry.go} (99%) rename core/{connection_test.go => retry_test.go} (99%) diff --git a/core/connection.go b/core/retry.go similarity index 99% rename from core/connection.go rename to core/retry.go index a0c37de09..ddf3c50b2 100644 --- a/core/connection.go +++ b/core/retry.go @@ -1,4 +1,4 @@ -package connection +package retry import ( "fmt" diff --git a/core/connection_test.go b/core/retry_test.go similarity index 99% rename from core/connection_test.go rename to core/retry_test.go index 3c909bc85..e6421d800 100644 --- a/core/connection_test.go +++ b/core/retry_test.go @@ -1,4 +1,4 @@ -package connection_test +package retry_test import ( "context" From b8bf783c17ae09b5e30712834d297ce4a85fe2a8 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 11:55:52 -0300 Subject: [PATCH 083/135] remove not found error for ProcessNewSignature --- aggregator/pkg/server.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 12c401d9d..72d55b217 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -130,7 +130,6 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // |---RETRYABLE---| -// Error throw is ______ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { var err error processNewSignature_func := func() error { @@ -139,11 +138,6 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd blsSignature, operatorId, ) if err != nil { - // Note return type will be nil - if err.Error() == "not found" { - err = connection.TransientError{Inner: err} - return err - } if strings.Contains(err.Error(), "connect: connection refused") { err = connection.TransientError{Inner: err} return err @@ -152,7 +146,7 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd err = connection.TransientError{Inner: err} return err } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = connection.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } return err } From 2e67ccbd47dc1f795a12a142646e11d61b633344 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 11:57:09 -0300 Subject: [PATCH 084/135] import retry functions from avs writer --- core/chainio/avs_writer.go | 121 ------------------------------------- 1 file changed, 121 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 978c5009d..4961c1b24 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/big" - "strings" "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients" @@ -16,7 +15,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" - connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/config" ) @@ -165,122 +163,3 @@ func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byt } return nil } - -// |---RETRYABLE---| - -func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { - var ( - tx *types.Transaction - err error - ) - respondToTaskV2_func := func() (*types.Transaction, error) { - tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return tx, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return tx, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - } - return tx, err - } - return connection.RetryWithData(respondToTaskV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int -}, error) { - var ( - state struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - } - err error - ) - batchesState_func := func() (struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - }, error) { - state, err = w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) - if err != nil { - state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) - // If error is not nil throw out result in state! - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return state, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return state, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - } - return state, err - } - return connection.RetryWithData(batchesState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { - var ( - batcherBalance *big.Int - err error - ) - batcherBalances_func := func() (*big.Int, error) { - batcherBalance, err = w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) - if err != nil { - batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return batcherBalance, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return batcherBalance, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - } - return batcherBalance, err - } - return connection.RetryWithData(batcherBalances_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { - var ( - aggregatorBalance *big.Int - err error - ) - balanceAt_func := func() (*big.Int, error) { - aggregatorBalance, err = w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) - if err != nil { - aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return aggregatorBalance, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return aggregatorBalance, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - } - return aggregatorBalance, err - } - return connection.RetryWithData(balanceAt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} From 3c437ece9f3c6711aacece4707ad2750e1e4d859 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 11:59:26 -0300 Subject: [PATCH 085/135] import retry functions in avs subscriber + only log success once --- core/chainio/avs_subscriber.go | 209 +-------------------------------- 1 file changed, 5 insertions(+), 204 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 5913e502e..e4252f2d4 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -3,24 +3,18 @@ package chainio import ( "context" "encoding/hex" - "fmt" - "math/big" - "strings" "sync" "time" - "github.com/ethereum/go-ethereum" ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" + retry "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/config" sdklogging "github.com/Layr-Labs/eigensdk-go/logging" "github.com/ethereum/go-ethereum/crypto" - connection "github.com/yetanotherco/aligned_layer/core" ) const ( @@ -71,17 +65,15 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) // Subscribe to new tasks - // NOTE: Should we consolidate these subscribers behind two interfaces sub, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks after %d retries", connection.NumRetries, "err", err) + s.logger.Error("Primary failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err } - s.logger.Info("Subscribed to new AlignedLayer V2 tasks") subFallback, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks %d retries", connection.NumRetries, "err", err) + s.logger.Error("Fallback failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err } s.logger.Info("Subscribed to new AlignedLayer V2 tasks") @@ -146,15 +138,13 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema // Subscribe to new tasks sub, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer tasks after %d retries", MaxRetries, "err", err) + s.logger.Error("Primary failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err } - s.logger.Info("Subscribed to new AlignedLayer V3 tasks") - //TODO: collapse this subFallback, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) if err != nil { - s.logger.Error("Failed to subscribe to new AlignedLayer %d tasks", MaxRetries, "err", err) + s.logger.Error("Fallback failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err } s.logger.Info("Subscribed to new AlignedLayer V3 tasks") @@ -399,192 +389,3 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { // func (s *AvsSubscriber) ParseTaskResponded(rawLog types.Log) (*cstaskmanager.ContractAlignedLayerTaskManagerTaskResponded, error) { // return s.AvsContractBindings.TaskManager.ContractAlignedLayerTaskManagerFilterer.ParseTaskResponded(rawLog) // } - -// |---RETRYABLE---| - -func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { - var ( - latestBlock uint64 - err error - ) - latestBlock_func := func() (uint64, error) { - latestBlock, err = s.AvsContractBindings.ethClient.BlockNumber(ctx) - if err != nil { - latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) - if err != nil { - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return latestBlock, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return latestBlock, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - } - return latestBlock, err - } - return connection.RetryWithData(latestBlock_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - var ( - logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator - err error - ) - - filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return logs, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return logs, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - return logs, err - } - return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - var ( - logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator - err error - ) - filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return logs, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return logs, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - return logs, err - } - return connection.RetryWithData(filterNewBatchV2_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int -}, error) { - var ( - state struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - } - err error - ) - batchState_func := func() (struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - }, error) { - state, err = s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return state, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return state, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - return state, err - } - - return connection.RetryWithData(batchState_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { - var ( - sub ethereum.Subscription - err error - ) - subscribeNewHead_func := func() (ethereum.Subscription, error) { - sub, err = s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) - if err != nil { - sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return sub, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return sub, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - } - return sub, err - } - return connection.RetryWithData(subscribeNewHead_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func SubscribeToNewTasksV2Retrayable( - serviceManager *servicemanager.ContractAlignedLayerServiceManager, - newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, - logger sdklogging.Logger, -) (event.Subscription, error) { - var ( - sub event.Subscription - err error - ) - subscribe_func := func() (event.Subscription, error) { - sub, err = serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) - if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return sub, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return sub, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - return sub, err - } - return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} - -func SubscribeToNewTasksV3Retryable( - serviceManager *servicemanager.ContractAlignedLayerServiceManager, - newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, - logger sdklogging.Logger, -) (event.Subscription, error) { - var ( - sub event.Subscription - err error - ) - subscribe_func := func() (event.Subscription, error) { - sub, err = serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) - if err != nil { - if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} - return sub, err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return sub, connection.TransientError{Inner: err} - } - err = connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} - } - return sub, err - } - return connection.RetryWithData(subscribe_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) -} From 718c343cb4ad326f19751eac69e0953a83f0da86 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 11:59:49 -0300 Subject: [PATCH 086/135] add retryable module to chainio --- core/chainio/retryable.go | 316 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 core/chainio/retryable.go diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go new file mode 100644 index 000000000..ca758e271 --- /dev/null +++ b/core/chainio/retryable.go @@ -0,0 +1,316 @@ +package chainio + +import ( + "context" + "fmt" + "math/big" + "strings" + + sdklogging "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" + retry "github.com/yetanotherco/aligned_layer/core" +) + +// |---AVS_WRITER---| + +func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { + var ( + tx *types.Transaction + err error + ) + respondToTaskV2_func := func() (*types.Transaction, error) { + tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + // Note return type will be nil + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return tx, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return tx, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + } + return tx, err + } + return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int +}, error) { + var ( + state struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + } + err error + ) + batchesState_func := func() (struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err = w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) + if err != nil { + state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return state, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return state, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + } + return state, err + } + return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { + var ( + batcherBalance *big.Int + err error + ) + batcherBalances_func := func() (*big.Int, error) { + batcherBalance, err = w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + if err != nil { + batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return batcherBalance, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return batcherBalance, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + } + return batcherBalance, err + } + return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { + var ( + aggregatorBalance *big.Int + err error + ) + balanceAt_func := func() (*big.Int, error) { + aggregatorBalance, err = w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) + if err != nil { + aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return aggregatorBalance, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return aggregatorBalance, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + } + return aggregatorBalance, err + } + return retry.RetryWithData(balanceAt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +// |---AVS_SUBSCRIBER---| + +func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { + var ( + latestBlock uint64 + err error + ) + latestBlock_func := func() (uint64, error) { + latestBlock, err = s.AvsContractBindings.ethClient.BlockNumber(ctx) + if err != nil { + latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return latestBlock, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return latestBlock, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + } + return latestBlock, err + } + return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + var ( + logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator + err error + ) + + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { + logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return logs, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return logs, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return logs, err + } + return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { + var ( + logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator + err error + ) + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { + logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return logs, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return logs, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return logs, err + } + return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int +}, error) { + var ( + state struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + } + err error + ) + batchState_func := func() (struct { + TaskCreatedBlock uint32 + Responded bool + RespondToTaskFeeLimit *big.Int + }, error) { + state, err = s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return state, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return state, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return state, err + } + + return retry.RetryWithData(batchState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { + var ( + sub ethereum.Subscription + err error + ) + subscribeNewHead_func := func() (ethereum.Subscription, error) { + sub, err = s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) + if err != nil { + sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return sub, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + } + return sub, err + } + return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func SubscribeToNewTasksV2Retrayable( + serviceManager *servicemanager.ContractAlignedLayerServiceManager, + newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, + logger sdklogging.Logger, +) (event.Subscription, error) { + var ( + sub event.Subscription + err error + ) + subscribe_func := func() (event.Subscription, error) { + sub, err = serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return sub, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return sub, err + } + return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} + +func SubscribeToNewTasksV3Retryable( + serviceManager *servicemanager.ContractAlignedLayerServiceManager, + newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, + logger sdklogging.Logger, +) (event.Subscription, error) { + var ( + sub event.Subscription + err error + ) + subscribe_func := func() (event.Subscription, error) { + sub, err = serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) + if err != nil { + if strings.Contains(err.Error(), "connect: retry refused") { + err = retry.TransientError{Inner: err} + return sub, err + } + if strings.Contains(err.Error(), "read: retry reset by peer") { + return sub, retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return sub, err + } + return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) +} From 920e8e4fdd8151d417d66998ec1128493e4105fe Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 12:10:58 -0300 Subject: [PATCH 087/135] rm comment + change permanent to transient error --- core/utils/eth_client_utils.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 9407c6745..b843c291d 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -9,14 +9,9 @@ import ( eigentypes "github.com/Layr-Labs/eigensdk-go/types" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - connection "github.com/yetanotherco/aligned_layer/core" + retry "github.com/yetanotherco/aligned_layer/core" ) -/* -Errors: -- "not found": (Transient) Call successfully returns but the tx receipt was not found. -- "connect: connection refused": (Transient) Could not connect. -*/ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { // For if no receipt and no error TransactionReceipt return "not found" as an error catch all ref: https://github.com/ethereum/go-ethereum/blob/master/ethclient/ethclient.go#L313 receipt_func := func() (*types.Receipt, error) { @@ -24,19 +19,19 @@ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx conte if err != nil { // Note return type will be nil if err.Error() == "not found" { - return nil, connection.TransientError{Inner: err} + return nil, retry.TransientError{Inner: err} } if strings.Contains(err.Error(), "connect: connection refused") { - return nil, connection.TransientError{Inner: err} + return nil, retry.TransientError{Inner: err} } if strings.Contains(err.Error(), "read: connection reset by peer") { - return nil, connection.TransientError{Inner: err} + return nil, retry.TransientError{Inner: err} } - return nil, connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + return nil, retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } return tx, err } - return connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) + return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) } func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { From ccc386bb9c8bcf0adf3130e3640974138f67a997 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 13:03:00 -0300 Subject: [PATCH 088/135] add MaxInterval parameter --- aggregator/pkg/aggregator.go | 20 ++++++++++++++++---- aggregator/pkg/server.go | 11 ++++++----- core/chainio/retryable.go | 22 +++++++++++----------- core/retry.go | 15 +++++++++++---- core/retry_test.go | 32 ++++++++++++++++---------------- core/utils/eth_client_utils.go | 2 +- 6 files changed, 61 insertions(+), 41 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 4c474da64..d94c6dc60 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -4,13 +4,14 @@ import ( "context" "encoding/hex" "fmt" + "strings" "sync" "time" gethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" - connection "github.com/yetanotherco/aligned_layer/core" + retry "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/metrics" sdkclients "github.com/Layr-Labs/eigensdk-go/chainio/clients" @@ -386,11 +387,22 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by // |---RETRYABLE---| func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { + var err error initilizeNewTask_func := func() error { - // Returns an error if the - return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + err = agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + if err != nil { + if strings.Contains(err.Error(), "connect: connection refused") { + err = retry.TransientError{Inner: err} + return err + } + if strings.Contains(err.Error(), "read: connection reset by peer") { + return retry.TransientError{Inner: err} + } + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + } + return err } - return connection.Retry(initilizeNewTask_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) + return retry.Retry(initilizeNewTask_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } // Long-lived goroutine that periodically checks and removes old Tasks from stored Maps diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 72d55b217..e662f2e9a 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -11,7 +11,7 @@ import ( "github.com/Layr-Labs/eigensdk-go/crypto/bls" eigentypes "github.com/Layr-Labs/eigensdk-go/types" - connection "github.com/yetanotherco/aligned_layer/core" + retry "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/types" ) @@ -57,6 +57,7 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t taskIndex := uint32(0) ok := false + // Increase to wait half a second longer // NOTE: Since this does not interact with a fallible connection waiting we use a different retry mechanism than the rest of the aggregator. for i := 0; i < waitForEventRetries; i++ { agg.taskMutex.Lock() @@ -139,17 +140,17 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd ) if err != nil { if strings.Contains(err.Error(), "connect: connection refused") { - err = connection.TransientError{Inner: err} + err = retry.TransientError{Inner: err} return err } if strings.Contains(err.Error(), "read: connection reset by peer") { - err = connection.TransientError{Inner: err} + err = retry.TransientError{Inner: err} return err } - err = connection.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} } return err } - return connection.Retry(processNewSignature_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) + return retry.Retry(processNewSignature_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index ca758e271..70381ffa2 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -41,7 +41,7 @@ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl } return tx, err } - return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { @@ -78,7 +78,7 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { } return state, err } - return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { @@ -103,7 +103,7 @@ func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big } return batcherBalance, err } - return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { @@ -128,7 +128,7 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co } return aggregatorBalance, err } - return retry.RetryWithData(balanceAt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(balanceAt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } // |---AVS_SUBSCRIBER---| @@ -155,7 +155,7 @@ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error } return latestBlock, err } - return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { @@ -178,7 +178,7 @@ func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Con } return logs, err } - return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { @@ -200,7 +200,7 @@ func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Con } return logs, err } - return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { @@ -235,7 +235,7 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte return state, err } - return retry.RetryWithData(batchState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(batchState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { @@ -260,7 +260,7 @@ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- } return sub, err } - return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func SubscribeToNewTasksV2Retrayable( @@ -286,7 +286,7 @@ func SubscribeToNewTasksV2Retrayable( } return sub, err } - return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func SubscribeToNewTasksV3Retryable( @@ -312,5 +312,5 @@ func SubscribeToNewTasksV3Retryable( } return sub, err } - return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } diff --git a/core/retry.go b/core/retry.go index ddf3c50b2..632aafd28 100644 --- a/core/retry.go +++ b/core/retry.go @@ -12,6 +12,10 @@ This retry library was inspired by and uses Cenk Alti (https://github.com/cenkal We would like to thank him for his great work. */ +/* +Note we use a custom Permanent error type for asserting Permanent Erros within the retry library. +We do not implement an explicit Transient error type and operate under the assumption that all errors that are not Permanent are Transient. +*/ type PermanentError struct { Inner error } @@ -42,9 +46,10 @@ const MinDelay = 1000 const RetryFactor = 2 const NumRetries = 3 const MaxInterval = 60000 +const MaxElapsedTime = 0 // Same as Retry only that the functionToRetry can return a value upon correct execution -func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, factor float64, maxTries uint64, maxInterval uint64) (T, error) { +func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, factor float64, maxTries uint64, maxInterval uint64, maxElapsedTime uint64) (T, error) { f := func() (T, error) { var ( val T @@ -75,7 +80,8 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(maxInterval)) - expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption) + maxElapsedTimeOption := backoff.WithMaxElapsedTime(time.Millisecond * time.Duration(maxElapsedTime)) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption, maxElapsedTimeOption) var maxRetriesBackoff backoff.BackOff if maxTries > 0 { @@ -93,7 +99,7 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa // from the configuration are reached, or until a `PermanentError` is returned. // The function to be retried should return `PermanentError` when the condition for stop retrying // is met. -func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64, maxInterval uint64) error { +func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64, maxInterval uint64, maxElapsedTime uint64) error { f := func() error { var err error func() { @@ -121,7 +127,8 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(maxInterval)) - expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption) + maxElapsedTimeOption := backoff.WithMaxElapsedTime(time.Millisecond * time.Duration(maxElapsedTime)) + expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption, maxElapsedTimeOption) var maxRetriesBackoff backoff.BackOff if maxTries > 0 { diff --git a/core/retry_test.go b/core/retry_test.go index e6421d800..1853eb144 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" - connection "github.com/yetanotherco/aligned_layer/core" + retry "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/chainio" "github.com/yetanotherco/aligned_layer/core/config" "github.com/yetanotherco/aligned_layer/core/utils" @@ -28,7 +28,7 @@ import ( func DummyFunction(x uint64) (uint64, error) { if x == 42 { - return 0, connection.PermanentError{Inner: fmt.Errorf("Permanent error!")} + return 0, retry.PermanentError{Inner: fmt.Errorf("Permanent error!")} } else if x < 42 { return 0, fmt.Errorf("Transient error!") } @@ -40,7 +40,7 @@ func TestRetryWithData(t *testing.T) { x, err := DummyFunction(43) return &x, err } - _, err := connection.RetryWithData(function, 1000, 2, 3, connection.MaxInterval) + _, err := retry.RetryWithData(function, 1000, 2, 3, retry.MaxInterval, retry.MaxElapsedTime) if err != nil { t.Errorf("Retry error!: %s", err) } @@ -51,7 +51,7 @@ func TestRetry(t *testing.T) { _, err := DummyFunction(43) return err } - err := connection.Retry(function, 1000, 2, 3, connection.MaxInterval) + err := retry.Retry(function, 1000, 2, 3, retry.MaxInterval, retry.MaxElapsedTime) if err != nil { t.Errorf("Retry error!: %s", err) } @@ -172,7 +172,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -340,7 +340,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -395,7 +395,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -445,7 +445,7 @@ func TestBlockNumber(t *testing.T) { _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -494,7 +494,7 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -543,7 +543,7 @@ func TestFilterBatchV3(t *testing.T) { _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -595,7 +595,7 @@ func TestBatchesStateSubscriber(t *testing.T) { _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -646,7 +646,7 @@ func TestSubscribeNewHead(t *testing.T) { _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -735,7 +735,7 @@ func TestRespondToTaskV2(t *testing.T) { _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("RespondToTaksV2 Emitted non-Transient error: %s\n", err) return } @@ -796,7 +796,7 @@ func TestBatchesStateWriter(t *testing.T) { _, err = avsWriter.BatchesStateRetryable(bytes) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("BatchesStateWriter Emitted non-Transient error: %s\n", err) return } @@ -849,7 +849,7 @@ func TestBalanceAt(t *testing.T) { _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("BalanceAt Emitted non-Transient error: %s\n", err) return } @@ -900,7 +900,7 @@ func TestBatchersBalances(t *testing.T) { _, err = avsWriter.BatcherBalancesRetryable(sender_address) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(connection.TransientError); !ok { + if _, ok := err.(retry.TransientError); !ok { fmt.Printf("BatchersBalances Emitted non-Transient error: %s\n", err) return } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index b843c291d..cc02dad62 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -31,7 +31,7 @@ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx conte } return tx, err } - return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval) + return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { From 71a074d95127fba3a655d89e1e74b8cc129d8e32 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 14:23:47 -0300 Subject: [PATCH 089/135] Remove Transient error type --- aggregator/pkg/aggregator.go | 10 +--- core/chainio/avs_writer.go | 2 +- core/chainio/retryable.go | 104 +++++---------------------------- core/retry.go | 21 ++----- core/retry_test.go | 37 ++++++------ core/utils/eth_client_utils.go | 13 +---- 6 files changed, 39 insertions(+), 148 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index d94c6dc60..7f03d2f30 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "strings" "sync" "time" @@ -391,14 +390,7 @@ func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreated initilizeNewTask_func := func() error { err = agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) if err != nil { - if strings.Contains(err.Error(), "connect: connection refused") { - err = retry.TransientError{Inner: err} - return err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return err } diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 4961c1b24..d9a9709c2 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -102,7 +102,7 @@ func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bin w.logger.Info("Simulated cost", "cost", simulatedCost) // Get RespondToTaskFeeLimit - batchState, err := w.BatchesStateRetryable(batchIdentifierHash) + batchState, err := w.BatchesStateRetryable(&bind.CallOpts{}, batchIdentifierHash) if err != nil { // Fallback also failed // Proceed to check values against simulated costs diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index 70381ffa2..d591f9f8d 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/big" - "strings" sdklogging "github.com/Layr-Labs/eigensdk-go/logging" "github.com/ethereum/go-ethereum" @@ -28,15 +27,7 @@ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl if err != nil { tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { - // Note return type will be nil - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return tx, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return tx, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } } return tx, err @@ -44,7 +35,7 @@ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { +func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int @@ -57,6 +48,7 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { } err error ) + batchesState_func := func() (struct { TaskCreatedBlock uint32 Responded bool @@ -66,14 +58,7 @@ func (w *AvsWriter) BatchesStateRetryable(arg0 [32]byte) (struct { if err != nil { state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return state, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return state, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } } return state, err @@ -91,14 +76,7 @@ func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big if err != nil { batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return batcherBalance, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return batcherBalance, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } } return batcherBalance, err @@ -116,14 +94,7 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co if err != nil { aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return aggregatorBalance, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return aggregatorBalance, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } } return aggregatorBalance, err @@ -143,14 +114,7 @@ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error if err != nil { latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return latestBlock, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return latestBlock, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } } return latestBlock, err @@ -167,14 +131,7 @@ func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Con filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return logs, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return logs, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return logs, err } @@ -189,14 +146,7 @@ func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Con filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return logs, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return logs, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return logs, err } @@ -223,14 +173,7 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte }, error) { state, err = s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return state, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return state, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return state, err } @@ -248,14 +191,7 @@ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- if err != nil { sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return sub, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return sub, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } } return sub, err @@ -275,14 +211,7 @@ func SubscribeToNewTasksV2Retrayable( subscribe_func := func() (event.Subscription, error) { sub, err = serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return sub, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return sub, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return sub, err } @@ -301,14 +230,7 @@ func SubscribeToNewTasksV3Retryable( subscribe_func := func() (event.Subscription, error) { sub, err = serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) if err != nil { - if strings.Contains(err.Error(), "connect: retry refused") { - err = retry.TransientError{Inner: err} - return sub, err - } - if strings.Contains(err.Error(), "read: retry reset by peer") { - return sub, retry.TransientError{Inner: err} - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return sub, err } diff --git a/core/retry.go b/core/retry.go index 632aafd28..a53b15ca5 100644 --- a/core/retry.go +++ b/core/retry.go @@ -29,19 +29,6 @@ func (e PermanentError) Is(err error) bool { return ok } -type TransientError struct { - Inner error -} - -func (e TransientError) Error() string { return e.Inner.Error() } -func (e TransientError) Unwrap() error { - return e.Inner -} -func (e TransientError) Is(err error) bool { - _, ok := err.(TransientError) - return ok -} - const MinDelay = 1000 const RetryFactor = 2 const NumRetries = 3 @@ -59,9 +46,9 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa defer func() { if r := recover(); r != nil { if panic_err, ok := r.(error); ok { - err = TransientError{panic_err} + err = panic_err } else { - err = TransientError{fmt.Errorf("panicked: %v", panic_err)} + err = fmt.Errorf("panicked: %v", panic_err) } } }() @@ -106,9 +93,9 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri defer func() { if r := recover(); r != nil { if panic_err, ok := r.(error); ok { - err = TransientError{panic_err} + err = panic_err } else { - err = TransientError{fmt.Errorf("panicked: %v", panic_err)} + err = fmt.Errorf("panicked: %v", panic_err) } } }() diff --git a/core/retry_test.go b/core/retry_test.go index 1853eb144..86900685f 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -15,6 +15,7 @@ import ( "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" rpccalls "github.com/Layr-Labs/eigensdk-go/metrics/collectors/rpc_calls" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/prometheus/client_golang/prometheus" @@ -171,8 +172,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -339,8 +339,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -394,8 +393,8 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + // If it retruend a permanent error we exit + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -445,7 +444,7 @@ func TestBlockNumber(t *testing.T) { _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -493,8 +492,8 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + // + if _, ok := err.(retry.PermanentError); !ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -543,7 +542,7 @@ func TestFilterBatchV3(t *testing.T) { _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -595,7 +594,7 @@ func TestBatchesStateSubscriber(t *testing.T) { _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -646,7 +645,7 @@ func TestSubscribeNewHead(t *testing.T) { _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } @@ -735,7 +734,7 @@ func TestRespondToTaskV2(t *testing.T) { _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("RespondToTaksV2 Emitted non-Transient error: %s\n", err) return } @@ -784,7 +783,7 @@ func TestBatchesStateWriter(t *testing.T) { var bytes [32]byte num.FillBytes(bytes[:]) - _, err = avsWriter.BatchesStateRetryable(bytes) + _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.Nil(t, err) // Kill Anvil at end of test @@ -793,10 +792,10 @@ func TestBatchesStateWriter(t *testing.T) { return } - _, err = avsWriter.BatchesStateRetryable(bytes) + _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("BatchesStateWriter Emitted non-Transient error: %s\n", err) return } @@ -811,7 +810,7 @@ func TestBatchesStateWriter(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BatchesStateRetryable(bytes) + _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.Nil(t, err) // Kill Anvil at end of test @@ -849,7 +848,7 @@ func TestBalanceAt(t *testing.T) { _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("BalanceAt Emitted non-Transient error: %s\n", err) return } @@ -900,7 +899,7 @@ func TestBatchersBalances(t *testing.T) { _, err = avsWriter.BatcherBalancesRetryable(sender_address) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. - if _, ok := err.(retry.TransientError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("BatchersBalances Emitted non-Transient error: %s\n", err) return } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index cc02dad62..4cbecec5f 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -3,7 +3,6 @@ package utils import ( "context" "fmt" - "strings" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" eigentypes "github.com/Layr-Labs/eigensdk-go/types" @@ -16,18 +15,10 @@ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx conte // For if no receipt and no error TransactionReceipt return "not found" as an error catch all ref: https://github.com/ethereum/go-ethereum/blob/master/ethclient/ethclient.go#L313 receipt_func := func() (*types.Receipt, error) { tx, err := client.TransactionReceipt(ctx, txHash) + // Note catch for all Permanent error. Defaults to returning if err != nil { // Note return type will be nil - if err.Error() == "not found" { - return nil, retry.TransientError{Inner: err} - } - if strings.Contains(err.Error(), "connect: connection refused") { - return nil, retry.TransientError{Inner: err} - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return nil, retry.TransientError{Inner: err} - } - return nil, retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + return nil, fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return tx, err } From 385bba32a2d6c690b7ca9c32dd9dbd79bce97bbe Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 14:45:49 -0300 Subject: [PATCH 090/135] nits for retry test with Transient error --- aggregator/pkg/server.go | 11 +---------- core/chainio/retryable.go | 3 ++- core/retry_test.go | 6 +++--- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index e662f2e9a..4f11c94f7 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "net/rpc" - "strings" "time" "github.com/Layr-Labs/eigensdk-go/crypto/bls" @@ -139,15 +138,7 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd blsSignature, operatorId, ) if err != nil { - if strings.Contains(err.Error(), "connect: connection refused") { - err = retry.TransientError{Inner: err} - return err - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - err = retry.TransientError{Inner: err} - return err - } - err = retry.TransientError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + err = fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err) } return err } diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index d591f9f8d..541736bad 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -131,7 +131,7 @@ func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Con filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) if err != nil { - fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) + err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) } return logs, err } @@ -212,6 +212,7 @@ func SubscribeToNewTasksV2Retrayable( sub, err = serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) if err != nil { err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) + return sub, err } return sub, err } diff --git a/core/retry_test.go b/core/retry_test.go index 86900685f..00e1028b1 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -159,7 +159,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { // Assert Call succeeds when Anvil running _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) - if err.Error() != "not found" { + if !strings.Contains(err.Error(), "not found") { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) return } @@ -189,7 +189,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err) - if err.Error() != "not found" { + if !strings.Contains(err.Error(), "not found") { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) return } @@ -493,7 +493,7 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) assert.NotNil(t, err) // - if _, ok := err.(retry.PermanentError); !ok { + if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } From f2d7d81da4a9b04853c3d6515815c7b85d08db4c Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 15:00:31 -0300 Subject: [PATCH 091/135] remove capture var --- aggregator/pkg/aggregator.go | 7 +- aggregator/pkg/server.go | 8 +- core/chainio/retryable.go | 141 +++------------------------------ core/utils/eth_client_utils.go | 10 +-- 4 files changed, 15 insertions(+), 151 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 7f03d2f30..bf00d1c4e 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -386,13 +386,8 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by // |---RETRYABLE---| func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { - var err error initilizeNewTask_func := func() error { - err = agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - return err + return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) } return retry.Retry(initilizeNewTask_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 4f11c94f7..c496de87c 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -3,7 +3,6 @@ package pkg import ( "context" "encoding/hex" - "fmt" "net/http" "net/rpc" "time" @@ -131,16 +130,11 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // |---RETRYABLE---| func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { - var err error processNewSignature_func := func() error { - err = agg.blsAggregationService.ProcessNewSignature( + return agg.blsAggregationService.ProcessNewSignature( ctx, taskIndex, taskResponse, blsSignature, operatorId, ) - if err != nil { - err = fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err) - } - return err } return retry.Retry(processNewSignature_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index 541736bad..f82e2f1ee 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -2,7 +2,6 @@ package chainio import ( "context" - "fmt" "math/big" sdklogging "github.com/Layr-Labs/eigensdk-go/logging" @@ -18,19 +17,8 @@ import ( // |---AVS_WRITER---| func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { - var ( - tx *types.Transaction - err error - ) respondToTaskV2_func := func() (*types.Transaction, error) { - tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - } - return tx, err + return w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) } return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -40,64 +28,26 @@ func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (s Responded bool RespondToTaskFeeLimit *big.Int }, error) { - var ( - state struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - } - err error - ) - batchesState_func := func() (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int }, error) { - state, err = w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) - if err != nil { - state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - } - return state, err + return w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) } return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { - var ( - batcherBalance *big.Int - err error - ) batcherBalances_func := func() (*big.Int, error) { - batcherBalance, err = w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) - if err != nil { - batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - } - return batcherBalance, err + return w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) } return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { - var ( - aggregatorBalance *big.Int - err error - ) balanceAt_func := func() (*big.Int, error) { - aggregatorBalance, err = w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) - if err != nil { - aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - } - return aggregatorBalance, err + return w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) } return retry.RetryWithData(balanceAt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -105,50 +55,23 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co // |---AVS_SUBSCRIBER---| func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { - var ( - latestBlock uint64 - err error - ) latestBlock_func := func() (uint64, error) { - latestBlock, err = s.AvsContractBindings.ethClient.BlockNumber(ctx) - if err != nil { - latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - } - return latestBlock, err + return s.AvsContractBindings.ethClient.BlockNumber(ctx) } return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - var ( - logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator - err error - ) - filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - return logs, err + return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - var ( - logs *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator - err error - ) + filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - logs, err = s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - return logs, err + return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -158,43 +81,20 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte Responded bool RespondToTaskFeeLimit *big.Int }, error) { - var ( - state struct { - TaskCreatedBlock uint32 - Responded bool - RespondToTaskFeeLimit *big.Int - } - err error - ) batchState_func := func() (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int }, error) { - state, err = s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - return state, err + return s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) } return retry.RetryWithData(batchState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { - var ( - sub ethereum.Subscription - err error - ) subscribeNewHead_func := func() (ethereum.Subscription, error) { - sub, err = s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) - if err != nil { - sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - } - return sub, err + return s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) } return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -204,17 +104,8 @@ func SubscribeToNewTasksV2Retrayable( newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, logger sdklogging.Logger, ) (event.Subscription, error) { - var ( - sub event.Subscription - err error - ) subscribe_func := func() (event.Subscription, error) { - sub, err = serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - return sub, err - } - return sub, err + return serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) } return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -224,16 +115,8 @@ func SubscribeToNewTasksV3Retryable( newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, logger sdklogging.Logger, ) (event.Subscription, error) { - var ( - sub event.Subscription - err error - ) subscribe_func := func() (event.Subscription, error) { - sub, err = serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) - if err != nil { - err = fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - return sub, err + return serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) } return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 4cbecec5f..8679edbc9 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -2,7 +2,6 @@ package utils import ( "context" - "fmt" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" eigentypes "github.com/Layr-Labs/eigensdk-go/types" @@ -12,15 +11,8 @@ import ( ) func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { - // For if no receipt and no error TransactionReceipt return "not found" as an error catch all ref: https://github.com/ethereum/go-ethereum/blob/master/ethclient/ethclient.go#L313 receipt_func := func() (*types.Receipt, error) { - tx, err := client.TransactionReceipt(ctx, txHash) - // Note catch for all Permanent error. Defaults to returning - if err != nil { - // Note return type will be nil - return nil, fmt.Errorf("Transient error: Unexpected Error while retrying: %s\n", err) - } - return tx, err + return client.TransactionReceipt(ctx, txHash) } return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } From 25cbbd867f6b49074f7b9127e5b874cf0448e752 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 7 Nov 2024 15:56:48 -0300 Subject: [PATCH 092/135] feat: bump only when timeout paased also refactors code to new retry implementation --- core/chainio/avs_writer.go | 92 ++++++++++++++++++++++++++++------ core/utils/eth_client_utils.go | 32 ++++-------- 2 files changed, 87 insertions(+), 37 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 8521df020..0d740b2b2 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -18,15 +18,16 @@ import ( servicemanager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" connection "github.com/yetanotherco/aligned_layer/core" "github.com/yetanotherco/aligned_layer/core/config" + "github.com/yetanotherco/aligned_layer/core/utils" ) const ( // How much to bump every retry (constant) GasBaseBumpPercentage int = 20 // An extra percentage to bump every retry i*5 (linear) - GasBumpIncrementalBumpPercentage int = 5 + GasBumpIncrementalPercentage int = 5 // Wait as much as 3 blocks time for the receipt - SendAggregateResponseReceiptTimeout time.Duration = time.Second * 36 + BlocksToWaitBeforeBump time.Duration = time.Second * 36 ) type AvsWriter struct { @@ -84,33 +85,94 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E // Sends AggregatedResponse and waits for the receipt for three blocks, if not received // it will try again bumping the last tx gas price based on `CalculateGasPriceBump` // This process happens indefinitely until the transaction is included. -// -// Note: If the rpc endpoints fail, the retry mechanism stops, returning a permanent error. -// This is because retries are infinite and we want to prevent increasing the time between them too much as it is exponential. -// And, if the rpc is down for a good period of time, we might fall into an infinite waiting. func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, onRetry func()) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction - tx, err := w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + + respondToTaskV2SimulationFunc := func() (*types.Transaction, error) { + tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + // check if reverted only else transient error + err = connection.PermanentError{Inner: fmt.Errorf("transaction reverted")} + } + } + return tx, err + } + tx, err := connection.RetryWithData(respondToTaskV2SimulationFunc, connection.MinDelay, connection.RetryFactor, 0, connection.MaxInterval) if err != nil { return nil, err } err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { - return nil, err + return nil, connection.PermanentError{Inner: err} } - // Set the nonce, as we might have to replace the transaction with a higher fee - txNonce := new(big.Int).SetUint64(tx.Nonce()) + // Set the nonce, as we might have to replace the transaction with a higher gas price + txNonce := big.NewInt(int64(tx.Nonce())) + txOpts.Nonce = txNonce txOpts.NoSend = false - txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit - tx, err = w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - return nil, err + i := 0 + + shouldBump := false + + respondToTaskV2Func := func() (*types.Receipt, error) { + // We bump only when the timeout for waiting for the receipt has passed + // not when an rpc failed + if shouldBump { + gasPrice, err := w.ClientFallback.SuggestGasPrice(context.Background()) + if err != nil { + shouldBump = false + return nil, connection.TransientError{Inner: err} + } + bumpedGasPrice := utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, GasBaseBumpPercentage, GasBumpIncrementalPercentage, i) + + if gasPrice.Cmp(txOpts.GasPrice) > 0 { + txOpts.GasPrice = bumpedGasPrice + } else { + txOpts.GasPrice = new(big.Int).Mul(txOpts.GasPrice, big.NewInt(1)) + } + } + + err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) + if err != nil { + shouldBump = false + // We bump the fee so much that the transaction cost is more expensive than the batcher fee limit + return nil, connection.PermanentError{Inner: err} + } + + tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + // check if reverted only to be permanent + err = connection.TransientError{Inner: err} + shouldBump = false + } + } + + receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, context.Background(), tx.Hash()) + if receipt != nil { + return receipt, nil + } + + // if we are here, it means we have reached the receipt waiting timeout + // so in the next iteration we try again by bumping the fee to make sure its included + if i > 0 { + onRetry() + } + i++ + + shouldBump = true + if err != nil { + return nil, connection.TransientError{Inner: err} + } + return nil, connection.TransientError{Inner: fmt.Errorf("transaction failed")} } - return connection.RetryWithData(sendTransaction, 1000, 2, 0) + return connection.RetryWithData(respondToTaskV2Func, 1000, 2, 0, 60) } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index ad3b8abb2..3857d327e 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/big" - "strings" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" eigentypes "github.com/Layr-Labs/eigensdk-go/types" @@ -13,31 +12,20 @@ import ( connection "github.com/yetanotherco/aligned_layer/core" ) -/* -Errors: -- "not found": (Transient) Call successfully returns but the tx receipt was not found. -- "connect: connection refused": (Transient) Could not connect. -*/ func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { - // For if no receipt and no error TransactionReceipt return "not found" as an error catch all ref: https://github.com/ethereum/go-ethereum/blob/master/ethclient/ethclient.go#L313 receipt_func := func() (*types.Receipt, error) { - tx, err := client.TransactionReceipt(ctx, txHash) - if err != nil { - // Note return type will be nil - if err.Error() == "not found" { - return nil, connection.TransientError{Inner: err} - } - if strings.Contains(err.Error(), "connect: connection refused") { - return nil, connection.TransientError{Inner: err} - } - if strings.Contains(err.Error(), "read: connection reset by peer") { - return nil, connection.TransientError{Inner: err} - } - return nil, connection.PermanentError{Inner: fmt.Errorf("Permanent error: Unexpected Error while retrying: %s\n", err)} + receipt, err := client.TransactionReceipt(ctx, txHash) + if err != nil || ctx.Err() != nil { + return nil, connection.TransientError{Inner: err} } - return tx, err + if receipt != nil { + return receipt, nil + } + + return nil, connection.TransientError{Inner: fmt.Errorf("receipt not found")} } - return connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, connection.NumRetries, connection.MaxInterval) + + return connection.RetryWithData(receipt_func, connection.MinDelay, connection.RetryFactor, 0, connection.MaxInterval) } func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { From d65709950cbe7cd3477e3137783c5e540d15cdbc Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 16:00:27 -0300 Subject: [PATCH 093/135] use time.duration for maxElapsedTime --- core/retry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/retry.go b/core/retry.go index a53b15ca5..24d65f51d 100644 --- a/core/retry.go +++ b/core/retry.go @@ -114,7 +114,7 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) multiplierOption := backoff.WithMultiplier(factor) maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(maxInterval)) - maxElapsedTimeOption := backoff.WithMaxElapsedTime(time.Millisecond * time.Duration(maxElapsedTime)) + maxElapsedTimeOption := backoff.WithMaxElapsedTime(time.Duration(maxElapsedTime)) expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption, maxElapsedTimeOption) var maxRetriesBackoff backoff.BackOff From 332ae57411b47a5d9c9278b438cac50c6fec70be Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 16:15:33 -0300 Subject: [PATCH 094/135] fix: removed the fallback rpc node accidentally --- core/chainio/retryable.go | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index f82e2f1ee..02b0818c2 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -18,7 +18,11 @@ import ( func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { respondToTaskV2_func := func() (*types.Transaction, error) { - return w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + } + return tx, err } return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -28,26 +32,39 @@ func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (s Responded bool RespondToTaskFeeLimit *big.Int }, error) { + batchesState_func := func() (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int }, error) { - return w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) + state, err := w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) + if err != nil { + state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) + } + return state, err } return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { batcherBalances_func := func() (*big.Int, error) { - return w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + if err != nil { + batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) + } + return batcherBalance, err } return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { balanceAt_func := func() (*big.Int, error) { - return w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) + aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) + if err != nil { + aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) + } + return aggregatorBalance, err } return retry.RetryWithData(balanceAt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -56,7 +73,11 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { latestBlock_func := func() (uint64, error) { - return s.AvsContractBindings.ethClient.BlockNumber(ctx) + latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) + if err != nil { + latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) + } + return latestBlock, err } return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -69,7 +90,6 @@ func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Con } func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) } @@ -94,7 +114,11 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { subscribeNewHead_func := func() (ethereum.Subscription, error) { - return s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) + sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) + if err != nil { + sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) + } + return sub, err } return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } From d92f3915efa47741edff365ccfee9d9e5b5483f6 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 7 Nov 2024 16:21:34 -0300 Subject: [PATCH 095/135] feat: timer to waitTrasactionReceipt --- core/chainio/avs_writer.go | 5 ++--- core/retry_test.go | 6 +++--- core/utils/eth_client_utils.go | 8 ++++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index df10b54d1..f25d22654 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -113,9 +113,8 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txNonce := big.NewInt(int64(tx.Nonce())) txOpts.Nonce = txNonce txOpts.NoSend = false - i := 0 - shouldBump := false + i := 0 respondToTaskV2Func := func() (*types.Receipt, error) { // We bump only when the timeout for waiting for the receipt has passed @@ -152,7 +151,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe } } - receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, context.Background(), tx.Hash()) + receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, context.Background(), tx.Hash(), BlocksToWaitBeforeBump) if receipt != nil { return receipt, nil } diff --git a/core/retry_test.go b/core/retry_test.go index 00e1028b1..ebf6b78fc 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -157,7 +157,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { } // Assert Call succeeds when Anvil running - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash, time.Second*45) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) if !strings.Contains(err.Error(), "not found") { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) @@ -170,7 +170,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash, time.Second*45) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) @@ -187,7 +187,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash, time.Second*45) assert.NotNil(t, err) if !strings.Contains(err.Error(), "not found") { fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index dd0de5035..998f32453 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -3,6 +3,7 @@ package utils import ( "context" "math/big" + "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" eigentypes "github.com/Layr-Labs/eigensdk-go/types" @@ -11,11 +12,14 @@ import ( retry "github.com/yetanotherco/aligned_layer/core" ) -func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { +// WaitForTransactionReceiptRetryable repeatedly attempts to fetch the transaction receipt for a given transaction hash. +// If the receipt is not found, the function will retry with exponential backoff until the specified `waitTimeout` duration is reached. +// If the receipt is still unavailable after `waitTimeout`, it will return an error. +func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash, waitTimeout time.Duration) (*types.Receipt, error) { receipt_func := func() (*types.Receipt, error) { return client.TransactionReceipt(ctx, txHash) } - return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, 0, retry.MaxInterval, uint64(waitTimeout)) } func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { From 318526f0c522566a29ba5244044fc22086115c4d Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 7 Nov 2024 17:38:06 -0300 Subject: [PATCH 096/135] feat: move constants to config file --- aggregator/pkg/aggregator.go | 2 +- config-files/config-aggregator.yaml | 3 +++ core/chainio/avs_writer.go | 39 ++++++++--------------------- core/config/aggregator.go | 9 +++++++ core/utils/eth_client_utils.go | 2 +- 5 files changed, 25 insertions(+), 30 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 2e27a90e1..fe1fabc57 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -302,7 +302,7 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) onRetry := func() { agg.metrics.IncBumpedGasPriceForAggregatedResponse() } - receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature, onRetry) + receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature, agg.AggregatorConfig.Aggregator.GasBaseBumpPercentage, agg.AggregatorConfig.Aggregator.GasBumpIncrementalPercentage, agg.AggregatorConfig.Aggregator.TimeToWaitBeforeBump, onRetry) if err != nil { agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Error sending aggregated response for batch %s. Error: %s", hex.EncodeToString(batchIdentifierHash[:]), err) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index dc258e272..f33ac0070 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -37,6 +37,9 @@ aggregator: garbage_collector_period: 2m #The period of the GC process. Suggested value for Prod: '168h' (7 days) garbage_collector_tasks_age: 20 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days) garbage_collector_tasks_interval: 10 #The interval of queried blocks to get an old batch. Suggested value for prod: '900' (3 hours) + gas_base_bump_percentage: 20 # How much to bump gas price when responding to task. Suggested value 20% + gas_bump_incremental_percentage: 5 # An extra percentage to bump every retry i*5 when responding to task. Suggested value 5% + time_to_wait_before_bump: 36s # The time to wait for the receipt when responding to task. Suggested value 36 seconds (3 blocks) ## Operator Configurations # operator: diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index f25d22654..8af825681 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -20,15 +20,6 @@ import ( "github.com/yetanotherco/aligned_layer/core/utils" ) -const ( - // How much to bump every retry (constant) - GasBaseBumpPercentage int = 20 - // An extra percentage to bump every retry i*5 (linear) - GasBumpIncrementalPercentage int = 5 - // Wait as much as 3 blocks time for the receipt - BlocksToWaitBeforeBump time.Duration = time.Second * 36 -) - type AvsWriter struct { *avsregistry.ChainWriter AvsContractBindings *AvsServiceBindings @@ -84,7 +75,7 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E // Sends AggregatedResponse and waits for the receipt for three blocks, if not received // it will try again bumping the last tx gas price based on `CalculateGasPriceBump` // This process happens indefinitely until the transaction is included. -func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, onRetry func()) (*types.Receipt, error) { +func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, gasBumpPercentage uint, gasBumpIncrementalPercentage uint, timeToWaitBeforeBump time.Duration, onRetry func()) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction @@ -113,30 +104,24 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txNonce := big.NewInt(int64(tx.Nonce())) txOpts.Nonce = txNonce txOpts.NoSend = false - shouldBump := false i := 0 respondToTaskV2Func := func() (*types.Receipt, error) { // We bump only when the timeout for waiting for the receipt has passed // not when an rpc failed - if shouldBump { - gasPrice, err := w.ClientFallback.SuggestGasPrice(context.Background()) - if err != nil { - shouldBump = false - return nil, err - } - bumpedGasPrice := utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, GasBaseBumpPercentage, GasBumpIncrementalPercentage, i) - - if gasPrice.Cmp(txOpts.GasPrice) > 0 { - txOpts.GasPrice = bumpedGasPrice - } else { - txOpts.GasPrice = new(big.Int).Mul(txOpts.GasPrice, big.NewInt(1)) - } + gasPrice, err := w.ClientFallback.SuggestGasPrice(context.Background()) + if err != nil { + return nil, err + } + bumpedGasPrice := utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, gasBumpPercentage, gasBumpIncrementalPercentage, i) + if gasPrice.Cmp(txOpts.GasPrice) > 0 { + txOpts.GasPrice = bumpedGasPrice + } else { + txOpts.GasPrice = new(big.Int).Mul(txOpts.GasPrice, big.NewInt(1)) } err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { - shouldBump = false // We bump the fee so much that the transaction cost is more expensive than the batcher fee limit return nil, retry.PermanentError{Inner: err} } @@ -146,12 +131,11 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { // check if reverted only to be permanent - shouldBump = false return nil, err } } - receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, context.Background(), tx.Hash(), BlocksToWaitBeforeBump) + receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, context.Background(), tx.Hash(), timeToWaitBeforeBump) if receipt != nil { return receipt, nil } @@ -163,7 +147,6 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe } i++ - shouldBump = true if err != nil { return nil, err } diff --git a/core/config/aggregator.go b/core/config/aggregator.go index e38e76b78..7142f62c6 100644 --- a/core/config/aggregator.go +++ b/core/config/aggregator.go @@ -24,6 +24,9 @@ type AggregatorConfig struct { GarbageCollectorPeriod time.Duration GarbageCollectorTasksAge uint64 GarbageCollectorTasksInterval uint64 + GasBaseBumpPercentage uint + GasBumpIncrementalPercentage uint + TimeToWaitBeforeBump time.Duration } } @@ -38,6 +41,9 @@ type AggregatorConfigFromYaml struct { GarbageCollectorPeriod time.Duration `yaml:"garbage_collector_period"` GarbageCollectorTasksAge uint64 `yaml:"garbage_collector_tasks_age"` GarbageCollectorTasksInterval uint64 `yaml:"garbage_collector_tasks_interval"` + GasBaseBumpPercentage uint `yaml:"gas_base_bump_percentage"` + GasBumpIncrementalPercentage uint `yaml:"gas_bump_incremental_percentage"` + TimeToWaitBeforeBump time.Duration `yaml:"time_to_wait_before_bump"` } `yaml:"aggregator"` } @@ -82,6 +88,9 @@ func NewAggregatorConfig(configFilePath string) *AggregatorConfig { GarbageCollectorPeriod time.Duration GarbageCollectorTasksAge uint64 GarbageCollectorTasksInterval uint64 + GasBaseBumpPercentage uint + GasBumpIncrementalPercentage uint + TimeToWaitBeforeBump time.Duration }(aggregatorConfigFromYaml.Aggregator), } } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 998f32453..7434c5513 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -41,7 +41,7 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e // Simple algorithm to calculate the gasPrice bump based on: // the currentGasPrice, a base bump percentage, a retry percentage, and the retry count. // Formula: currentGasPrice + (currentGasPrice * (baseBumpPercentage + retryCount * incrementalRetryPercentage) / 100) -func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercentage int, retryAttemptPercentage int, retryCount int) *big.Int { +func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercentage uint, retryAttemptPercentage uint, retryCount int) *big.Int { // Incremental percentage increase for each retry attempt (i*5%) incrementalRetryPercentage := new(big.Int).Mul(big.NewInt(int64(retryAttemptPercentage)), big.NewInt(int64(retryCount))) From ce8dcd0030acc9d6e771f24a377af4983380cf28 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 7 Nov 2024 17:47:28 -0300 Subject: [PATCH 097/135] fix: bump fee 10% if gas price decreased from prev bumped gas price --- core/chainio/avs_writer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 8af825681..1bd308696 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -117,7 +117,9 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe if gasPrice.Cmp(txOpts.GasPrice) > 0 { txOpts.GasPrice = bumpedGasPrice } else { - txOpts.GasPrice = new(big.Int).Mul(txOpts.GasPrice, big.NewInt(1)) + bumpAmount := new(big.Int).Mul(txOpts.GasPrice, big.NewInt(10)) + bumpAmount = new(big.Int).Div(bumpAmount, big.NewInt(100)) + txOpts.GasPrice = new(big.Int).Mul(txOpts.GasPrice, bumpAmount) } err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) From 8d116c78470a6a46f6f3b41e646d2a36e6c1d3b5 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Thu, 7 Nov 2024 17:53:12 -0300 Subject: [PATCH 098/135] pass retryable functoin params explicitly from the calling function --- aggregator/pkg/aggregator.go | 2 +- core/chainio/avs_subscriber.go | 21 +++++++++++---------- core/chainio/avs_writer.go | 2 +- core/chainio/retryable.go | 29 +++++++++++++++-------------- core/retry_test.go | 32 ++++++++++++++++---------------- 5 files changed, 44 insertions(+), 42 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index bf00d1c4e..c6ffa5442 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -387,7 +387,7 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { initilizeNewTask_func := func() error { - return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, timeToExpiry) } return retry.Retry(initilizeNewTask_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index e4252f2d4..9aada4a27 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/accounts/abi/bind" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -65,13 +66,13 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) // Subscribe to new tasks - sub, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err := SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { s.logger.Error("Primary failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err } - subFallback, err := SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err := SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { s.logger.Error("Fallback failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err @@ -113,14 +114,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err = SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = SubscribeToNewTasksV2Retrayable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err = SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { errorChannel <- err } @@ -136,13 +137,13 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) // Subscribe to new tasks - sub, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err := SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { s.logger.Error("Primary failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err } - subFallback, err := SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err := SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { s.logger.Error("Fallback failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err @@ -184,14 +185,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManager, internalChannel, s.logger) + sub, err = SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = SubscribeToNewTasksV3Retryable(s.AvsContractBindings.ServiceManagerFallback, internalChannel, s.logger) + subFallback, err = SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { errorChannel <- err } @@ -270,7 +271,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag fromBlock = latestBlock - BlockInterval } - logs, err := s.FilterBatchV2Retryable(fromBlock, context.Background()) + logs, err := s.FilterBatchV2Retryable(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) if err != nil { return nil, err } @@ -319,7 +320,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag fromBlock = latestBlock - BlockInterval } - logs, err := s.FilterBatchV3Retryable(fromBlock, context.Background()) + logs, err := s.FilterBatchV3Retryable(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) if err != nil { return nil, err } diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index d9a9709c2..c887c1c0d 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -151,7 +151,7 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byte) error { // Get batcher balance - batcherBalance, err := w.BatcherBalancesRetryable(senderAddress) + batcherBalance, err := w.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) if err != nil { // Ignore and continue. w.logger.Error("Failed to get batcherBalance", "error", err) diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index 02b0818c2..33e668afc 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -4,7 +4,6 @@ import ( "context" "math/big" - sdklogging "github.com/Layr-Labs/eigensdk-go/logging" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -38,20 +37,20 @@ func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (s Responded bool RespondToTaskFeeLimit *big.Int }, error) { - state, err := w.AvsContractBindings.ServiceManager.BatchesState(&bind.CallOpts{}, arg0) + state, err := w.AvsContractBindings.ServiceManager.BatchesState(opts, arg0) if err != nil { - state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(&bind.CallOpts{}, arg0) + state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(opts, arg0) } return state, err } return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BatcherBalancesRetryable(senderAddress common.Address) (*big.Int, error) { +func (w *AvsWriter) BatcherBalancesRetryable(opts *bind.CallOpts, senderAddress common.Address) (*big.Int, error) { batcherBalances_func := func() (*big.Int, error) { - batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(&bind.CallOpts{}, senderAddress) + batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(opts, senderAddress) if err != nil { - batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(&bind.CallOpts{}, senderAddress) + batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(opts, senderAddress) } return batcherBalance, err } @@ -82,16 +81,16 @@ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) FilterBatchV2Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { +func (s *AvsSubscriber) FilterBatchV2Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(opts, batchMerkleRoot) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) FilterBatchV3Retryable(fromBlock uint64, ctx context.Context) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { +func (s *AvsSubscriber) FilterBatchV3Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { - return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: ctx}, nil) + return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(opts, batchMerkleRoot) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } @@ -124,23 +123,25 @@ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- } func SubscribeToNewTasksV2Retrayable( + opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, - logger sdklogging.Logger, + batchMerkleRoot [][32]byte, ) (event.Subscription, error) { subscribe_func := func() (event.Subscription, error) { - return serviceManager.WatchNewBatchV2(&bind.WatchOpts{}, newTaskCreatedChan, nil) + return serviceManager.WatchNewBatchV2(opts, newTaskCreatedChan, batchMerkleRoot) } return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } func SubscribeToNewTasksV3Retryable( + opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, - logger sdklogging.Logger, + batchMerkleRoot [][32]byte, ) (event.Subscription, error) { subscribe_func := func() (event.Subscription, error) { - return serviceManager.WatchNewBatchV3(&bind.WatchOpts{}, newTaskCreatedChan, nil) + return serviceManager.WatchNewBatchV3(opts, newTaskCreatedChan, batchMerkleRoot) } return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } diff --git a/core/retry_test.go b/core/retry_test.go index 00e1028b1..55a964f83 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -328,7 +328,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -337,7 +337,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { return } - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) @@ -354,7 +354,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV3Retryable(s.ServiceManager, channel, baseConfig.Logger) + _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -382,7 +382,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) + _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -391,7 +391,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { return } - _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) + _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) // If it retruend a permanent error we exit if _, ok := err.(retry.PermanentError); ok { @@ -409,7 +409,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV2Retrayable(s.ServiceManager, channel, baseConfig.Logger) + _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -481,7 +481,7 @@ func TestFilterBatchV2(t *testing.T) { if err != nil { return } - _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) + _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -490,7 +490,7 @@ func TestFilterBatchV2(t *testing.T) { return } - _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) + _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) // if _, ok := err.(retry.PermanentError); ok { @@ -508,7 +508,7 @@ func TestFilterBatchV2(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.FilterBatchV2Retryable(0, context.Background()) + _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -530,7 +530,7 @@ func TestFilterBatchV3(t *testing.T) { if err != nil { return } - _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) + _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -539,7 +539,7 @@ func TestFilterBatchV3(t *testing.T) { return } - _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) + _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { @@ -557,7 +557,7 @@ func TestFilterBatchV3(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.FilterBatchV3Retryable(0, context.Background()) + _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) // Kill Anvil at end of test @@ -885,9 +885,9 @@ func TestBatchersBalances(t *testing.T) { if err != nil { return } - sender_address := common.HexToAddress("0x0") + senderAddress := common.HexToAddress("0x0") - _, err = avsWriter.BatcherBalancesRetryable(sender_address) + _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) // Kill Anvil at end of test @@ -896,7 +896,7 @@ func TestBatchersBalances(t *testing.T) { return } - _, err = avsWriter.BatcherBalancesRetryable(sender_address) + _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.NotNil(t, err) // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { @@ -914,7 +914,7 @@ func TestBatchersBalances(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BatcherBalancesRetryable(sender_address) + _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) // Kill Anvil at end of test From bbe16b2227ace4520e4ec2623ba66d662b9fd617 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 7 Nov 2024 17:53:30 -0300 Subject: [PATCH 099/135] fix: go ci --- core/utils/eth_client_utils_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/utils/eth_client_utils_test.go b/core/utils/eth_client_utils_test.go index 72d718ed2..5c56d2793 100644 --- a/core/utils/eth_client_utils_test.go +++ b/core/utils/eth_client_utils_test.go @@ -8,8 +8,11 @@ import ( ) func TestCalculateGasPriceBumpBasedOnRetry(t *testing.T) { - baseBumpPercentage := 20 - incrementalRetryPercentage := 5 + var incrementalRetryPercentage uint = 20 + var baseBumpPercentage uint = 5 + + baseBumpPercentage = 20 + incrementalRetryPercentage = 5 gasPrices := [5]*big.Int{ big.NewInt(3000000000), big.NewInt(3000000000), From 320821a050e0fb5279753960f7f33c3a673bf61e Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 7 Nov 2024 18:45:59 -0300 Subject: [PATCH 100/135] fix: go ci --- core/utils/eth_client_utils_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils/eth_client_utils_test.go b/core/utils/eth_client_utils_test.go index 5c56d2793..cd13337ab 100644 --- a/core/utils/eth_client_utils_test.go +++ b/core/utils/eth_client_utils_test.go @@ -8,8 +8,8 @@ import ( ) func TestCalculateGasPriceBumpBasedOnRetry(t *testing.T) { - var incrementalRetryPercentage uint = 20 - var baseBumpPercentage uint = 5 + incrementalRetryPercentage := uint(20) + baseBumpPercentage := uint(5) baseBumpPercentage = 20 incrementalRetryPercentage = 5 From 976753692e34d74554dfaeb2ea5418b615e7a10f Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Thu, 7 Nov 2024 19:00:16 -0300 Subject: [PATCH 101/135] fix: go ci --- core/utils/eth_client_utils_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/utils/eth_client_utils_test.go b/core/utils/eth_client_utils_test.go index cd13337ab..11d57150f 100644 --- a/core/utils/eth_client_utils_test.go +++ b/core/utils/eth_client_utils_test.go @@ -11,8 +11,6 @@ func TestCalculateGasPriceBumpBasedOnRetry(t *testing.T) { incrementalRetryPercentage := uint(20) baseBumpPercentage := uint(5) - baseBumpPercentage = 20 - incrementalRetryPercentage = 5 gasPrices := [5]*big.Int{ big.NewInt(3000000000), big.NewInt(3000000000), From d1c17e106f60f5846e1ff9042a606fed52fce56e Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 01:17:23 -0300 Subject: [PATCH 102/135] use duration in retry function --- core/retry.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/core/retry.go b/core/retry.go index 24d65f51d..c7cbf92ac 100644 --- a/core/retry.go +++ b/core/retry.go @@ -29,14 +29,16 @@ func (e PermanentError) Is(err error) bool { return ok } -const MinDelay = 1000 -const RetryFactor = 2 -const NumRetries = 3 -const MaxInterval = 60000 -const MaxElapsedTime = 0 +const ( + MinDelay = 1 * time.Second + MaxInterval = 60 * time.Second + MaxElapsedTime = 0 * time.Second + RetryFactor float64 = 2 + NumRetries uint64 = 3 +) // Same as Retry only that the functionToRetry can return a value upon correct execution -func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, factor float64, maxTries uint64, maxInterval uint64, maxElapsedTime uint64) (T, error) { +func RetryWithData[T any](functionToRetry func() (T, error), minDelay time.Duration, factor float64, maxTries uint64, maxInterval time.Duration, maxElapsedTime time.Duration) (T, error) { f := func() (T, error) { var ( val T @@ -64,10 +66,10 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa randomOption := backoff.WithRandomizationFactor(0) - initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) + initialRetryOption := backoff.WithInitialInterval(minDelay) multiplierOption := backoff.WithMultiplier(factor) - maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(maxInterval)) - maxElapsedTimeOption := backoff.WithMaxElapsedTime(time.Millisecond * time.Duration(maxElapsedTime)) + maxIntervalOption := backoff.WithMaxInterval(maxInterval) + maxElapsedTimeOption := backoff.WithMaxElapsedTime(maxElapsedTime) expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption, maxElapsedTimeOption) var maxRetriesBackoff backoff.BackOff @@ -86,7 +88,7 @@ func RetryWithData[T any](functionToRetry func() (T, error), minDelay uint64, fa // from the configuration are reached, or until a `PermanentError` is returned. // The function to be retried should return `PermanentError` when the condition for stop retrying // is met. -func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTries uint64, maxInterval uint64, maxElapsedTime uint64) error { +func Retry(functionToRetry func() error, minDelay time.Duration, factor float64, maxTries uint64, maxInterval time.Duration, maxElapsedTime time.Duration) error { f := func() error { var err error func() { @@ -111,10 +113,10 @@ func Retry(functionToRetry func() error, minDelay uint64, factor float64, maxTri randomOption := backoff.WithRandomizationFactor(0) - initialRetryOption := backoff.WithInitialInterval(time.Millisecond * time.Duration(minDelay)) + initialRetryOption := backoff.WithInitialInterval(minDelay) multiplierOption := backoff.WithMultiplier(factor) - maxIntervalOption := backoff.WithMaxInterval(time.Millisecond * time.Duration(maxInterval)) - maxElapsedTimeOption := backoff.WithMaxElapsedTime(time.Duration(maxElapsedTime)) + maxIntervalOption := backoff.WithMaxInterval(maxInterval) + maxElapsedTimeOption := backoff.WithMaxElapsedTime(maxElapsedTime) expBackoff := backoff.NewExponentialBackOff(randomOption, multiplierOption, initialRetryOption, maxIntervalOption, maxElapsedTimeOption) var maxRetriesBackoff backoff.BackOff From 84055b5c2005ecfc9ee3ee07968c337bfa52c86e Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 04:08:38 -0300 Subject: [PATCH 103/135] add retry logic to getTaskIndex + remove test comments --- aggregator/pkg/server.go | 39 +++++----- core/retry_test.go | 161 ++++++++++++++------------------------- 2 files changed, 77 insertions(+), 123 deletions(-) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index c496de87c..bb97e4d9b 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -3,6 +3,7 @@ package pkg import ( "context" "encoding/hex" + "fmt" "net/http" "net/rpc" "time" @@ -13,9 +14,6 @@ import ( "github.com/yetanotherco/aligned_layer/core/types" ) -const waitForEventRetries = 50 -const waitForEventSleepSeconds = 4 * time.Second - func (agg *Aggregator) ServeOperators() error { // Registers a new RPC server err := rpc.Register(agg) @@ -53,24 +51,10 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t "BatchIdentifierHash", "0x"+hex.EncodeToString(signedTaskResponse.BatchIdentifierHash[:]), "operatorId", hex.EncodeToString(signedTaskResponse.OperatorId[:])) taskIndex := uint32(0) - ok := false - // Increase to wait half a second longer - // NOTE: Since this does not interact with a fallible connection waiting we use a different retry mechanism than the rest of the aggregator. - for i := 0; i < waitForEventRetries; i++ { - agg.taskMutex.Lock() - agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Starting processing of Response") - taskIndex, ok = agg.batchesIdxByIdentifierHash[signedTaskResponse.BatchIdentifierHash] - if !ok { - agg.taskMutex.Unlock() - agg.logger.Info("- Unlocked Resources: Task not found in the internal map") - time.Sleep(waitForEventSleepSeconds) - } else { - break - } - } + taskIndex, err := agg.GetTaskIndexRetryable(signedTaskResponse.BatchIdentifierHash) - if !ok { + if err != nil { agg.logger.Warn("Task not found in the internal map, operator signature will be lost. Batch may not reach quorum") *reply = 1 return nil @@ -139,3 +123,20 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd return retry.Retry(processNewSignature_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } + +func (agg *Aggregator) GetTaskIndexRetryable(batchIdentifierHash [32]byte) (uint32, error) { + getTaskIndex_func := func() (uint32, error) { + agg.taskMutex.Lock() + agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Starting processing of Response") + taskIndex, ok := agg.batchesIdxByIdentifierHash[batchIdentifierHash] + if !ok { + agg.taskMutex.Unlock() + agg.logger.Info("- Unlocked Resources: Task not found in the internal map") + return taskIndex, fmt.Errorf("Task not found in the internal map") + } else { + return taskIndex, nil + } + } + + return retry.RetryWithData(getTaskIndex_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) +} diff --git a/core/retry_test.go b/core/retry_test.go index 55a964f83..c0ce88e92 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -98,7 +98,6 @@ func SetupAnvil(port uint16) (*exec.Cmd, *eth.InstrumentedClient, error) { } func TestAnvilSetupKill(t *testing.T) { - // Start Anvil cmd, _, err := SetupAnvil(8545) if err != nil { log.Fatal("Error setting up Anvil: ", err) @@ -113,7 +112,6 @@ func TestAnvilSetupKill(t *testing.T) { err = p.Signal(syscall.Signal(0)) assert.Nil(t, err, "Anvil Process Killed") - // Kill Anvil if err := cmd.Process.Kill(); err != nil { fmt.Printf("Error killing process: %v\n", err) return @@ -133,10 +131,8 @@ func TestAnvilSetupKill(t *testing.T) { // |--Aggreagator Retry Tests--| -// Waits for receipt from anvil node -> Will fail to get receipt func TestWaitForTransactionReceiptRetryable(t *testing.T) { - // Retry call Params to := common.BytesToAddress([]byte{0x11}) tx := types.NewTx(&types.AccessListTx{ ChainID: big.NewInt(1337), @@ -150,13 +146,11 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { hash := tx.Hash() - // Start anvil cmd, client, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } - // Assert Call succeeds when Anvil running _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) if !strings.Contains(err.Error(), "not found") { @@ -164,7 +158,6 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } - // Kill Anvil if err = cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -181,7 +174,6 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } - // Start anvil cmd, client, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -194,23 +186,23 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return } } +// NOTE: The following tests involving starting the aggregator panic after the connection to anvil is cut crashing the test runner. +// The originates within the eigen-sdk and as of 8/11/24 is currently working to be fixed. + /* func TestInitializeNewTaskRetryable(t *testing.T) { - //Start Anvil _, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } - //Start Aggregator aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") agg, err := aggregator.NewAggregator(*aggregatorConfig) if err != nil { @@ -220,51 +212,73 @@ func TestInitializeNewTaskRetryable(t *testing.T) { quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(byte(0))} quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(byte(57))} - // Should succeed with err msg err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) assert.Nil(t, err) - // TODO: Find exact error to assert - - // Kill Anvil - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - time.Sleep(2 * time.Second) - - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) - - // Start Anvil - _, _, err = SetupAnvil(8545) - if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) - } - - // Should succeed - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) - assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) - // Kill Anvil - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) - return - } - time.Sleep(2 * time.Second) + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } + + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.NotNil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + _, _, err = SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + assert.Nil(t, err) + fmt.Printf("Error setting Avs Subscriber: %s\n", err) + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } +} +*/ + +/* +func TestGetTaskIndexRetryable(t *testing.T) { + + cmd, _, err := SetupAnvil(8545) + if err != nil { + fmt.Printf("Error setting up Anvil: %s\n", err) + } + + aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") + agg, err := aggregator.NewAggregator(*aggregatorConfig) + if err != nil { + aggregatorConfig.BaseConfig.Logger.Error("Cannot create aggregator", "err", err) + return + } + zero_bytes := [32]byte{} + + // Task is not present in map should return transient error + _, err = agg.GetTaskIndexRetryable(zero_bytes) + assert.NotNil(t, err) + if !strings.Contains(err.Error(), "Task not found in the internal map") { + fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + return + } + + if err := cmd.Process.Kill(); err != nil { + fmt.Printf("error killing process: %v\n", err) + return + } } */ /* // |--Server Retry Tests--| func TestProcessNewSignatureRetryable(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) } - //Start Aggregator aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") agg, err := aggregator.NewAggregator(*aggregatorConfig) if err != nil { @@ -277,20 +291,17 @@ func TestProcessNewSignatureRetryable(t *testing.T) { err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.NotNil(t, err) - // TODO: Find exact error to assert // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return } - time.Sleep(2 * time.Second) err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.NotNil(t, err) fmt.Printf("Error Processing New Signature: %s\n", err) - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -300,7 +311,6 @@ func TestProcessNewSignatureRetryable(t *testing.T) { assert.Nil(t, err) fmt.Printf("Error Processing New Signature: %s\n", err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -311,7 +321,6 @@ func TestProcessNewSignatureRetryable(t *testing.T) { // |--AVS-Subscriber Retry Tests--| func TestSubscribeToNewTasksV3Retryable(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -331,7 +340,6 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -348,7 +356,6 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -357,7 +364,6 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -365,7 +371,6 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { } func TestSubscribeToNewTasksV2(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -385,7 +390,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -393,7 +397,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) - // If it retruend a permanent error we exit if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return @@ -403,7 +406,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -412,7 +414,6 @@ func TestSubscribeToNewTasksV2(t *testing.T) { _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -426,7 +427,6 @@ func TestBlockNumber(t *testing.T) { fmt.Printf("Error setting up Anvil: %s\n", err) } - //channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") sub, err := chainio.NewAvsSubscriberFromConfig(aggregatorConfig.BaseConfig) if err != nil { @@ -435,7 +435,6 @@ func TestBlockNumber(t *testing.T) { _, err = sub.BlockNumberRetryable(context.Background()) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -443,7 +442,6 @@ func TestBlockNumber(t *testing.T) { _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return @@ -453,7 +451,6 @@ func TestBlockNumber(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -462,7 +459,6 @@ func TestBlockNumber(t *testing.T) { _, err = sub.BlockNumberRetryable(context.Background()) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -470,7 +466,6 @@ func TestBlockNumber(t *testing.T) { } func TestFilterBatchV2(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -484,7 +479,6 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -492,7 +486,6 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) - // if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return @@ -502,7 +495,6 @@ func TestFilterBatchV2(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -511,7 +503,6 @@ func TestFilterBatchV2(t *testing.T) { _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -519,7 +510,6 @@ func TestFilterBatchV2(t *testing.T) { } func TestFilterBatchV3(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -533,7 +523,6 @@ func TestFilterBatchV3(t *testing.T) { _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -541,7 +530,6 @@ func TestFilterBatchV3(t *testing.T) { _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return @@ -551,7 +539,6 @@ func TestFilterBatchV3(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -560,7 +547,6 @@ func TestFilterBatchV3(t *testing.T) { _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -568,7 +554,6 @@ func TestFilterBatchV3(t *testing.T) { } func TestBatchesStateSubscriber(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -582,10 +567,8 @@ func TestBatchesStateSubscriber(t *testing.T) { zero_bytes := [32]byte{} _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) - //TODO: Find exact failure error assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -593,7 +576,6 @@ func TestBatchesStateSubscriber(t *testing.T) { _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return @@ -603,7 +585,6 @@ func TestBatchesStateSubscriber(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -612,7 +593,6 @@ func TestBatchesStateSubscriber(t *testing.T) { _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -620,7 +600,6 @@ func TestBatchesStateSubscriber(t *testing.T) { } func TestSubscribeNewHead(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -636,7 +615,6 @@ func TestSubscribeNewHead(t *testing.T) { _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -644,7 +622,6 @@ func TestSubscribeNewHead(t *testing.T) { _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return @@ -654,7 +631,6 @@ func TestSubscribeNewHead(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -663,7 +639,6 @@ func TestSubscribeNewHead(t *testing.T) { _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -693,8 +668,6 @@ func TestRespondToTaskV2(t *testing.T) { g1Point, g1Point, g1Point, } - // Or if you want to initialize with specific values - nonSignerStakesAndSignature := servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature{ NonSignerPubkeys: g1Points, QuorumApks: g1Points, @@ -725,7 +698,6 @@ func TestRespondToTaskV2(t *testing.T) { return } - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -733,7 +705,6 @@ func TestRespondToTaskV2(t *testing.T) { _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("RespondToTaksV2 Emitted non-Transient error: %s\n", err) return @@ -743,7 +714,6 @@ func TestRespondToTaskV2(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -758,7 +728,6 @@ func TestRespondToTaskV2(t *testing.T) { return } - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -766,7 +735,6 @@ func TestRespondToTaskV2(t *testing.T) { } func TestBatchesStateWriter(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -786,7 +754,6 @@ func TestBatchesStateWriter(t *testing.T) { _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -794,7 +761,6 @@ func TestBatchesStateWriter(t *testing.T) { _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("BatchesStateWriter Emitted non-Transient error: %s\n", err) return @@ -804,7 +770,6 @@ func TestBatchesStateWriter(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -813,7 +778,6 @@ func TestBatchesStateWriter(t *testing.T) { _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -821,7 +785,6 @@ func TestBatchesStateWriter(t *testing.T) { } func TestBalanceAt(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -832,14 +795,12 @@ func TestBalanceAt(t *testing.T) { if err != nil { return } - //TODO: Source Aggregator Address aggregator_address := common.HexToAddress("0x0") blockHeight := big.NewInt(13) _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -847,7 +808,6 @@ func TestBalanceAt(t *testing.T) { _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("BalanceAt Emitted non-Transient error: %s\n", err) return @@ -857,7 +817,6 @@ func TestBalanceAt(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -866,7 +825,6 @@ func TestBalanceAt(t *testing.T) { _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -874,7 +832,6 @@ func TestBalanceAt(t *testing.T) { } func TestBatchersBalances(t *testing.T) { - // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -890,7 +847,6 @@ func TestBatchersBalances(t *testing.T) { _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return @@ -898,7 +854,6 @@ func TestBatchersBalances(t *testing.T) { _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.NotNil(t, err) - // Assert returned error is both transient error and contains the expected error msg. if _, ok := err.(retry.PermanentError); ok { fmt.Printf("BatchersBalances Emitted non-Transient error: %s\n", err) return @@ -908,7 +863,6 @@ func TestBatchersBalances(t *testing.T) { return } - // Start anvil cmd, _, err = SetupAnvil(8545) if err != nil { fmt.Printf("Error setting up Anvil: %s\n", err) @@ -917,7 +871,6 @@ func TestBatchersBalances(t *testing.T) { _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) - // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { fmt.Printf("error killing process: %v\n", err) return From bc63ef9b7d36c0d794391e125ef319845fed6e63 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 04:14:23 -0300 Subject: [PATCH 104/135] test comment nits --- core/retry_test.go | 80 +++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/core/retry_test.go b/core/retry_test.go index c0ce88e92..58323f2ce 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -159,7 +159,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { } if err = cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } @@ -187,7 +187,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { } if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -341,18 +341,18 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) return } @@ -365,7 +365,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -391,18 +391,18 @@ func TestSubscribeToNewTasksV2(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) return } @@ -415,7 +415,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -436,18 +436,18 @@ func TestBlockNumber(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("BlockNumber Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("BlockNumber Emitted non Transient error: %s\n", err) return } @@ -460,7 +460,7 @@ func TestBlockNumber(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -480,18 +480,18 @@ func TestFilterBatchV2(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("FilterBatchV2 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("FilterBatchV2 Emitted non Transient error: %s\n", err) return } @@ -504,7 +504,7 @@ func TestFilterBatchV2(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -524,18 +524,18 @@ func TestFilterBatchV3(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("FilerBatchV3 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("FilterBatchV3 Emitted non Transient error: %s\n", err) return } @@ -548,7 +548,7 @@ func TestFilterBatchV3(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -570,18 +570,18 @@ func TestBatchesStateSubscriber(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) return } @@ -594,7 +594,7 @@ func TestBatchesStateSubscriber(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -616,18 +616,18 @@ func TestSubscribeNewHead(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("SubscribeNewHead Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + fmt.Printf("SubscribeNewHead Emitted non Transient error: %s\n", err) return } @@ -640,7 +640,7 @@ func TestSubscribeNewHead(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -682,7 +682,7 @@ func TestRespondToTaskV2(t *testing.T) { aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") w, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) if err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } txOpts := *w.Signer.GetTxOpts() @@ -699,14 +699,14 @@ func TestRespondToTaskV2(t *testing.T) { } if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("RespondToTaksV2 Emitted non-Transient error: %s\n", err) + fmt.Printf("RespondToTaskV2 Emitted non-Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { @@ -729,7 +729,7 @@ func TestRespondToTaskV2(t *testing.T) { } if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -743,7 +743,7 @@ func TestBatchesStateWriter(t *testing.T) { aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") avsWriter, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) if err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } num := big.NewInt(6) @@ -779,7 +779,7 @@ func TestBatchesStateWriter(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -802,7 +802,7 @@ func TestBalanceAt(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } @@ -826,7 +826,7 @@ func TestBalanceAt(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } @@ -848,7 +848,7 @@ func TestBatchersBalances(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } @@ -872,7 +872,7 @@ func TestBatchersBalances(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + fmt.Printf("Error killing process: %v\n", err) return } } From 57090dafa33649faf021c28028ad08b2199c3dbb Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 05:57:00 -0300 Subject: [PATCH 105/135] emit refactor unit tests --- core/retry_test.go | 208 ++++++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 108 deletions(-) diff --git a/core/retry_test.go b/core/retry_test.go index 58323f2ce..c9883fcbe 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -15,6 +15,7 @@ import ( "github.com/Layr-Labs/eigensdk-go/chainio/clients/eth" rpccalls "github.com/Layr-Labs/eigensdk-go/metrics/collectors/rpc_calls" + backoff "github.com/cenkalti/backoff/v4" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -113,7 +114,7 @@ func TestAnvilSetupKill(t *testing.T) { assert.Nil(t, err, "Anvil Process Killed") if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } @@ -148,46 +149,46 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { cmd, client, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) if !strings.Contains(err.Error(), "not found") { - fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) + t.Errorf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) return } if err = cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + t.Errorf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + t.Errorf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } cmd, client, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) assert.NotNil(t, err) if !strings.Contains(err.Error(), "not found") { - fmt.Printf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) + t.Errorf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) return } if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -200,7 +201,7 @@ func TestInitializeNewTaskRetryable(t *testing.T) { _, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -216,25 +217,25 @@ func TestInitializeNewTaskRetryable(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + t.Errorf("error killing process: %v\n", err) return } err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) assert.NotNil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + t.Errorf("Error setting Avs Subscriber: %s\n", err) _, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) assert.Nil(t, err) - fmt.Printf("Error setting Avs Subscriber: %s\n", err) + t.Errorf("Error setting Avs Subscriber: %s\n", err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + t.Errorf("error killing process: %v\n", err) return } } @@ -245,7 +246,7 @@ func TestGetTaskIndexRetryable(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -260,12 +261,12 @@ func TestGetTaskIndexRetryable(t *testing.T) { _, err = agg.GetTaskIndexRetryable(zero_bytes) assert.NotNil(t, err) if !strings.Contains(err.Error(), "Task not found in the internal map") { - fmt.Printf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) + t.Errorf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) return } if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + t.Errorf("error killing process: %v\n", err) return } } @@ -276,7 +277,7 @@ func TestGetTaskIndexRetryable(t *testing.T) { func TestProcessNewSignatureRetryable(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -294,25 +295,25 @@ func TestProcessNewSignatureRetryable(t *testing.T) { // Kill Anvil at end of test if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + t.Errorf("error killing process: %v\n", err) return } err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.NotNil(t, err) - fmt.Printf("Error Processing New Signature: %s\n", err) + t.Errorf("Error Processing New Signature: %s\n", err) cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.Nil(t, err) - fmt.Printf("Error Processing New Signature: %s\n", err) + t.Errorf("Error Processing New Signature: %s\n", err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + t.Errorf("error killing process: %v\n", err) return } } @@ -323,7 +324,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { func TestSubscribeToNewTasksV3Retryable(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) @@ -334,38 +335,38 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { baseConfig.AlignedLayerDeploymentConfig.AlignedLayerOperatorStateRetrieverAddr, baseConfig.EthWsClient, baseConfig.EthWsClientFallback, baseConfig.Logger) if err != nil { - fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) + t.Errorf("Error setting up Avs Service Bindings: %s\n", err) } _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) + t.Errorf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) + t.Errorf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -373,7 +374,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { func TestSubscribeToNewTasksV2(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } channel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) @@ -384,38 +385,38 @@ func TestSubscribeToNewTasksV2(t *testing.T) { baseConfig.AlignedLayerDeploymentConfig.AlignedLayerOperatorStateRetrieverAddr, baseConfig.EthWsClient, baseConfig.EthWsClientFallback, baseConfig.Logger) if err != nil { - fmt.Printf("Error setting up Avs Service Bindings: %s\n", err) + t.Errorf("Error setting up Avs Service Bindings: %s\n", err) } _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) + t.Errorf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) + t.Errorf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -424,7 +425,7 @@ func TestBlockNumber(t *testing.T) { // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -436,31 +437,31 @@ func TestBlockNumber(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("BlockNumber Emitted non Transient error: %s\n", err) + t.Errorf("BlockNumber Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("BlockNumber Emitted non Transient error: %s\n", err) + t.Errorf("BlockNumber Emitted non Transient error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = sub.BlockNumberRetryable(context.Background()) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -468,7 +469,7 @@ func TestBlockNumber(t *testing.T) { func TestFilterBatchV2(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -480,31 +481,31 @@ func TestFilterBatchV2(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("FilterBatchV2 Emitted non Transient error: %s\n", err) + t.Errorf("FilterBatchV2 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("FilterBatchV2 Emitted non Transient error: %s\n", err) + t.Errorf("FilterBatchV2 Emitted non Transient error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -512,7 +513,7 @@ func TestFilterBatchV2(t *testing.T) { func TestFilterBatchV3(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -524,31 +525,31 @@ func TestFilterBatchV3(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("FilerBatchV3 Emitted non Transient error: %s\n", err) + t.Errorf("FilerBatchV3 Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("FilterBatchV3 Emitted non Transient error: %s\n", err) + t.Errorf("FilterBatchV3 Emitted non Transient error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -556,7 +557,7 @@ func TestFilterBatchV3(t *testing.T) { func TestBatchesStateSubscriber(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -570,31 +571,31 @@ func TestBatchesStateSubscriber(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) + t.Errorf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) + t.Errorf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -602,7 +603,7 @@ func TestBatchesStateSubscriber(t *testing.T) { func TestSubscribeNewHead(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } c := make(chan *types.Header) @@ -616,32 +617,31 @@ func TestSubscribeNewHead(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("SubscribeNewHead Emitted non Transient error: %s\n", err) + t.Errorf("SubscribeNewHead Emitted non Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("SubscribeNewHead Emitted non Transient error: %s\n", err) + t.Errorf("SubscribeNewHead Emitted non Transient error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) - return + t.Errorf("Error killing process: %v\n", err) } } @@ -651,7 +651,7 @@ func TestRespondToTaskV2(t *testing.T) { // Start anvil cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } g2Point := servicemanager.BN254G2Point{ @@ -682,7 +682,7 @@ func TestRespondToTaskV2(t *testing.T) { aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") w, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) if err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } txOpts := *w.Signer.GetTxOpts() @@ -692,58 +692,50 @@ func TestRespondToTaskV2(t *testing.T) { // NOTE: With zero bytes the tx reverts _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) - // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" - if !strings.Contains(err.Error(), "execution reverted: custom error 0x2396d34e:") { - t.Errorf("Respond to task V2 Retryable did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") - return + if !strings.Contains(err.Error(), "execution reverted") { + t.Errorf("RespondToTaskV2 did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") } if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) - return + t.Errorf("Error killing process: %v\n", err) } _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) - if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("RespondToTaskV2 Emitted non-Transient error: %s\n", err) - return + if _, ok := err.(*backoff.PermanentError); ok { + t.Errorf("RespondToTaskV2 Emitted non-Transient error: %s\n", err) } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("RespondToTaskV2 did not return expected error: %s\n", err) - return + t.Errorf("RespondToTaskV2 did not return expected error: %s\n", err) } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } // NOTE: With zero bytes the tx reverts _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) - // assert error contains "Message:"execution reverted: custom error 0x2396d34e:" - if !strings.Contains(err.Error(), "execution reverted: custom error 0x2396d34e:") { - t.Errorf("Respond to task V2 Retryable did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") - return + if !strings.Contains(err.Error(), "execution reverted") { + t.Errorf("RespondToTaskV2 did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") } if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) - return + t.Errorf("Error killing process: %v\n", err) } } func TestBatchesStateWriter(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") avsWriter, err := chainio.NewAvsWriterFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig) if err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } num := big.NewInt(6) @@ -755,31 +747,31 @@ func TestBatchesStateWriter(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error killing process: %v\n", err) + t.Errorf("error killing process: %v\n", err) return } _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("BatchesStateWriter Emitted non-Transient error: %s\n", err) + t.Errorf("BatchesStateWriter Emitted non-Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("BatchesStateWriter did not contain expected error: %s\n", err) + t.Errorf("BatchesStateWriter did not contain expected error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -787,7 +779,7 @@ func TestBatchesStateWriter(t *testing.T) { func TestBalanceAt(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -802,31 +794,31 @@ func TestBalanceAt(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("BalanceAt Emitted non-Transient error: %s\n", err) + t.Errorf("BalanceAt Emitted non-Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("BalanceAt did not return expected error: %s\n", err) + t.Errorf("BalanceAt did not return expected error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } @@ -834,7 +826,7 @@ func TestBalanceAt(t *testing.T) { func TestBatchersBalances(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml") @@ -848,31 +840,31 @@ func TestBatchersBalances(t *testing.T) { assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { - fmt.Printf("BatchersBalances Emitted non-Transient error: %s\n", err) + t.Errorf("BatchersBalances Emitted non-Transient error: %s\n", err) return } if !strings.Contains(err.Error(), "connect: connection refused") { - fmt.Printf("BatchersBalances did not return expected error: %s\n", err) + t.Errorf("BatchersBalances did not return expected error: %s\n", err) return } cmd, _, err = SetupAnvil(8545) if err != nil { - fmt.Printf("Error setting up Anvil: %s\n", err) + t.Errorf("Error setting up Anvil: %s\n", err) } _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { - fmt.Printf("Error killing process: %v\n", err) + t.Errorf("Error killing process: %v\n", err) return } } From 103eace818d2db931f21a75a285f92c37e7fdd65 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 05:57:24 -0300 Subject: [PATCH 106/135] add permanent error for revert in RespondToTaskV2 --- core/chainio/retryable.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index 33e668afc..c5eca3d3b 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -3,6 +3,7 @@ package chainio import ( "context" "math/big" + "strings" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -16,10 +17,19 @@ import ( // |---AVS_WRITER---| func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { + var ( + tx *types.Transaction + err error + ) respondToTaskV2_func := func() (*types.Transaction, error) { - tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + if strings.Contains(err.Error(), "execution reverted:") { + err = retry.PermanentError{Inner: err} + } + } } return tx, err } From 7edc6bba0632537854deacefc20ffb42273c319a Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 06:04:19 -0300 Subject: [PATCH 107/135] rm cmt values for aggregator test yaml --- config-files/config-aggregator-test.yaml | 27 +----------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/config-files/config-aggregator-test.yaml b/config-files/config-aggregator-test.yaml index 31f934f41..ac0a122b7 100644 --- a/config-files/config-aggregator-test.yaml +++ b/config-files/config-aggregator-test.yaml @@ -19,14 +19,6 @@ bls: private_key_store_path: "../config-files/anvil.aggregator.bls.key.json" private_key_store_password: "" -# ## Batcher configurations # batcher: -# block_interval: 3 -# batch_size_interval: 10 -# max_proof_size: 67108864 # 64 MiB -# max_batch_size: 268435456 # 256 MiB -# eth_ws_reconnects: 99999999999999 -# pre_verification_is_enabled: true - ## Aggregator Configurations aggregator: server_ip_port_address: localhost:8090 @@ -34,21 +26,4 @@ aggregator: avs_service_manager_address: 0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690 enable_metrics: false metrics_ip_port_address: localhost:9091 - telemetry_ip_port_address: localhost:4001 -## Operator Configurations -# operator: -# aggregator_rpc_server_ip_port_address: localhost:8090 -# address: 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 -# earnings_receiver_address: 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 -# delegation_approver_address: "0x0000000000000000000000000000000000000000" -# staker_opt_out_window_blocks: 0 -# metadata_url: "https://yetanotherco.github.io/operator_metadata/metadata.json" -# enable_metrics: true -# metrics_ip_port_address: localhost:9092 -# max_batch_size: 268435456 # 256 MiB -# # Operators variables needed for register it in EigenLayer -# el_delegation_manager_address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" -# private_key_store_path: config-files/anvil.ecdsa.key.json -# bls_private_key_store_path: config-files/anvil.bls.key.json -# signer_type: local_keystore -# chain_id: 31337 + telemetry_ip_port_address: localhost:4001 \ No newline at end of file From eaf405bf39ae503cda67c8412c8f4f39ce47ffa4 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Fri, 8 Nov 2024 15:17:42 -0300 Subject: [PATCH 108/135] feat: bump gas price traces --- aggregator/pkg/aggregator.go | 6 +++++- aggregator/pkg/telemetry.go | 15 +++++++++++++++ core/chainio/avs_writer.go | 12 +++++++----- telemetry_api/lib/telemetry_api/traces.ex | 18 ++++++++++++++++++ .../controllers/trace_controller.ex | 15 +++++++++++++++ telemetry_api/lib/telemetry_api_web/router.ex | 1 + 6 files changed, 61 insertions(+), 6 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index fe1fabc57..34e62b961 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "math/big" "sync" "time" @@ -301,7 +302,10 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "senderAddress", hex.EncodeToString(senderAddress[:]), "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) - onRetry := func() { agg.metrics.IncBumpedGasPriceForAggregatedResponse() } + onRetry := func(bumpedGasPrice *big.Int) { + agg.metrics.IncBumpedGasPriceForAggregatedResponse() + agg.telemetry.BumpedTaskGasPrice(batchMerkleRoot, bumpedGasPrice.String()) + } receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature, agg.AggregatorConfig.Aggregator.GasBaseBumpPercentage, agg.AggregatorConfig.Aggregator.GasBumpIncrementalPercentage, agg.AggregatorConfig.Aggregator.TimeToWaitBeforeBump, onRetry) if err != nil { agg.walletMutex.Unlock() diff --git a/aggregator/pkg/telemetry.go b/aggregator/pkg/telemetry.go index dc73e20b0..95962d0fb 100644 --- a/aggregator/pkg/telemetry.go +++ b/aggregator/pkg/telemetry.go @@ -30,6 +30,11 @@ type TaskErrorMessage struct { TaskError string `json:"error"` } +type TaskGasPriceBumpMessage struct { + MerkleRoot string `json:"merkle_root"` + BumpedGasPrice string `json:"bumped_gas_price"` +} + type TaskSentToEthereumMessage struct { MerkleRoot string `json:"merkle_root"` TxHash string `json:"tx_hash"` @@ -96,6 +101,16 @@ func (t *Telemetry) LogTaskError(batchMerkleRoot [32]byte, taskError error) { } } +func (t *Telemetry) BumpedTaskGasPrice(batchMerkleRoot [32]byte, bumpedGasPrice string) { + body := TaskGasPriceBumpMessage{ + MerkleRoot: fmt.Sprintf("0x%s", hex.EncodeToString(batchMerkleRoot[:])), + BumpedGasPrice: bumpedGasPrice, + } + if err := t.sendTelemetryMessage("/api/aggregatorTaskGasPriceBump", body); err != nil { + t.logger.Error("[Telemetry] Error in LogOperatorResponse", "error", err) + } +} + func (t *Telemetry) TaskSentToEthereum(batchMerkleRoot [32]byte, txHash string) { body := TaskSentToEthereumMessage{ MerkleRoot: fmt.Sprintf("0x%s", hex.EncodeToString(batchMerkleRoot[:])), diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 1bd308696..5869f641c 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -75,7 +75,7 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E // Sends AggregatedResponse and waits for the receipt for three blocks, if not received // it will try again bumping the last tx gas price based on `CalculateGasPriceBump` // This process happens indefinitely until the transaction is included. -func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, gasBumpPercentage uint, gasBumpIncrementalPercentage uint, timeToWaitBeforeBump time.Duration, onRetry func()) (*types.Receipt, error) { +func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, gasBumpPercentage uint, gasBumpIncrementalPercentage uint, timeToWaitBeforeBump time.Duration, onRetry func(*big.Int)) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction @@ -114,13 +114,18 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, err } bumpedGasPrice := utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, gasBumpPercentage, gasBumpIncrementalPercentage, i) + // new bumped gas price must be higher than the last one (this should hardly ever happen though) if gasPrice.Cmp(txOpts.GasPrice) > 0 { txOpts.GasPrice = bumpedGasPrice } else { + // bump the last price 10% to replace it. bumpAmount := new(big.Int).Mul(txOpts.GasPrice, big.NewInt(10)) bumpAmount = new(big.Int).Div(bumpAmount, big.NewInt(100)) txOpts.GasPrice = new(big.Int).Mul(txOpts.GasPrice, bumpAmount) } + if i > 0 { + onRetry(txOpts.GasPrice) + } err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { @@ -143,10 +148,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe } // if we are here, it means we have reached the receipt waiting timeout - // so in the next iteration we try again by bumping the fee to make sure its included - if i > 0 { - onRetry() - } + // we increment the i here to add an incremental percentage to increase the odds of being included in the next blocks i++ if err != nil { diff --git a/telemetry_api/lib/telemetry_api/traces.ex b/telemetry_api/lib/telemetry_api/traces.ex index 2b8f529fd..e611f386a 100644 --- a/telemetry_api/lib/telemetry_api/traces.ex +++ b/telemetry_api/lib/telemetry_api/traces.ex @@ -208,6 +208,7 @@ defmodule TelemetryApi.Traces do :ok end end + @doc """ Registers the sending of a batcher task to Ethereum in the task trace. @@ -297,6 +298,23 @@ defmodule TelemetryApi.Traces do :ok end end + + @doc """ + Registers a bump in the gas price when the aggregator tries to respond to a task in the task trace. + + ## Examples + + iex> merkle_root + iex> bumped_gas_price + iex> aggregator_task_gas_price_bumped(merkle_root, bumped_gas_price) + :ok + """ + def aggregator_task_gas_price_bumped(merkle_root, bumped_gas_price) do + with {:ok, _trace} <- set_current_trace_with_subspan(merkle_root, :aggregator) do + Tracer.add_event("Task gas price bumped", [{"bumped__gas_price", bumped_gas_price}]) + :ok + end + end @doc """ Registers the sending of an aggregator task to Ethereum in the task trace. diff --git a/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex b/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex index 8968a05e2..210e4b667 100644 --- a/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex +++ b/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex @@ -121,6 +121,21 @@ defmodule TelemetryApiWeb.TraceController do |> render(:show_merkle, merkle_root: merkle_root) end end + + @doc """ + Registers a gas price bump in the trace of the given merkle_root + Method: POST aggregatorTaskGasPriceBump + """ + def aggregator_task_gas_price_bump(conn, %{ + "merkle_root" => merkle_root, + "bumped_gas_price" => bumped_gas_price + }) do + with :ok <- Traces.aggregator_task_gas_price_bumped(merkle_root, bumped_gas_price) do + conn + |> put_status(:ok) + |> render(:show_merkle, merkle_root: merkle_root) + end + end @doc """ Register a task sent, from the aggregator, to Ethereum in the trace of the given merkle_root diff --git a/telemetry_api/lib/telemetry_api_web/router.ex b/telemetry_api/lib/telemetry_api_web/router.ex index f60046417..200e6c94f 100644 --- a/telemetry_api/lib/telemetry_api_web/router.ex +++ b/telemetry_api/lib/telemetry_api_web/router.ex @@ -15,6 +15,7 @@ defmodule TelemetryApiWeb.Router do post "/operatorResponse", TraceController, :register_operator_response post "/quorumReached", TraceController, :quorum_reached post "/taskError", TraceController, :task_error + post "/aggregatorTaskGasPriceBump", TraceController, :aggregator_task_gas_price_bumped post "/aggregatorTaskSent", TraceController, :aggregator_task_sent post "/finishTaskTrace", TraceController, :finish_task_trace From 525001cc2cb0310d6c1790173477833356e85b19 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Fri, 8 Nov 2024 15:28:44 -0300 Subject: [PATCH 109/135] fix: permanent errs only when tx reverted --- core/chainio/avs_writer.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 5869f641c..62ac07a24 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "strings" "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients" @@ -84,8 +85,12 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe if err != nil { tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { - // check if reverted only else transient error - err = retry.PermanentError{Inner: fmt.Errorf("transaction reverted")} + if strings.Contains(err.Error(), "reverted") { + return nil, retry.PermanentError{Inner: err} + } else { + return nil, err + } + } } return tx, err @@ -137,8 +142,11 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe if err != nil { tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { - // check if reverted only to be permanent - return nil, err + if strings.Contains(err.Error(), "reverted") { + return nil, retry.PermanentError{Inner: err} + } else { + return nil, err + } } } From 7bfca755e790d4f1d46e83dfb43cf87a8d9c5ce3 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 15:56:57 -0300 Subject: [PATCH 110/135] add make target + add unit tests to ci --- .github/workflows/build-and-test-go.yml | 1 - .github/workflows/test-go-retries.yml | 34 +++++++++++++++++++++++++ Makefile | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-go-retries.yml diff --git a/.github/workflows/build-and-test-go.yml b/.github/workflows/build-and-test-go.yml index 51ed5fe85..7abf54ec4 100644 --- a/.github/workflows/build-and-test-go.yml +++ b/.github/workflows/build-and-test-go.yml @@ -38,4 +38,3 @@ jobs: run: go build operator/cmd/main.go - name: Build aggregator run: go build aggregator/cmd/main.go - diff --git a/.github/workflows/test-go-retries.yml b/.github/workflows/test-go-retries.yml new file mode 100644 index 000000000..2954253e7 --- /dev/null +++ b/.github/workflows/test-go-retries.yml @@ -0,0 +1,34 @@ +name: test-go-retries + +on: + push: + branches: [main] + pull_request: + branches: ["*"] + paths: + - 'core/**' + - '.github/workflows/test-go-retries.yml' + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Clear device space + run: | + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + cache: false + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Test go Retry Functions + run: make test_go_retries \ No newline at end of file diff --git a/Makefile b/Makefile index f7ad1e6e4..86b07625e 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,9 @@ aggregator_send_dummy_responses: @echo "Sending dummy responses to Aggregator..." @cd aggregator && go run dummy/submit_task_responses.go +test_go_retries: + @cd core/ && \ + go test -v __OPERATOR__: From 4eb7ba85c3d241613e004ffdd211398e407f15ce Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 16:05:57 -0300 Subject: [PATCH 111/135] add foundry to ci --- .github/workflows/test-go-retries.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-go-retries.yml b/.github/workflows/test-go-retries.yml index 2954253e7..79f1cfe97 100644 --- a/.github/workflows/test-go-retries.yml +++ b/.github/workflows/test-go-retries.yml @@ -30,5 +30,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: toolchain: stable + - name: foundry-toolchain + uses: foundry-rs/foundry-toolchain@v1.2.0 - name: Test go Retry Functions run: make test_go_retries \ No newline at end of file From d2f8d2dc14d7b78b626f7dcdc28ec58e6a87be72 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Fri, 8 Nov 2024 16:35:40 -0300 Subject: [PATCH 112/135] refactor: use respond to task retryabale --- core/chainio/avs_writer.go | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 62ac07a24..12dc3b282 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -80,22 +80,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction - respondToTaskV2SimulationFunc := func() (*types.Transaction, error) { - tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - if strings.Contains(err.Error(), "reverted") { - return nil, retry.PermanentError{Inner: err} - } else { - return nil, err - } - - } - } - return tx, err - } - tx, err := retry.RetryWithData(respondToTaskV2SimulationFunc, retry.MinDelay, retry.RetryFactor, 0, retry.MaxInterval, retry.MaxElapsedTime) + tx, err := w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err } @@ -138,16 +123,9 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, retry.PermanentError{Inner: err} } - tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - if strings.Contains(err.Error(), "reverted") { - return nil, retry.PermanentError{Inner: err} - } else { - return nil, err - } - } + tx, err = w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if strings.Contains(err.Error(), "reverted") { + return nil, err } receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, context.Background(), tx.Hash(), timeToWaitBeforeBump) From 550c9c24bf1254d02f4a0f8ff970f3ecd33b678e Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 16:36:52 -0300 Subject: [PATCH 113/135] change retry function names --- aggregator/pkg/aggregator.go | 2 +- aggregator/pkg/server.go | 8 +-- core/chainio/avs_subscriber.go | 32 ++++++------ core/chainio/avs_writer.go | 10 ++-- core/chainio/retryable.go | 22 ++++---- core/retry_test.go | 96 +++++++++++++++++----------------- core/utils/eth_client_utils.go | 2 +- 7 files changed, 86 insertions(+), 86 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index c6ffa5442..a8fb6c644 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -312,7 +312,7 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Sending aggregated response for batch %s", hex.EncodeToString(batchIdentifierHash[:])) - receipt, err := utils.WaitForTransactionReceiptRetryable( + receipt, err := utils.WaitForTransactionReceipt( agg.AggregatorConfig.BaseConfig.EthRpcClient, context.Background(), *txHash) if err != nil { agg.telemetry.LogTaskError(batchMerkleRoot, err) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index bb97e4d9b..5e716d332 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -52,7 +52,7 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t "operatorId", hex.EncodeToString(signedTaskResponse.OperatorId[:])) taskIndex := uint32(0) - taskIndex, err := agg.GetTaskIndexRetryable(signedTaskResponse.BatchIdentifierHash) + taskIndex, err := agg.GetTaskIndex(signedTaskResponse.BatchIdentifierHash) if err != nil { agg.logger.Warn("Task not found in the internal map, operator signature will be lost. Batch may not reach quorum") @@ -71,7 +71,7 @@ func (agg *Aggregator) ProcessOperatorSignedTaskResponseV2(signedTaskResponse *t agg.logger.Info("Starting bls signature process") go func() { - err := agg.ProcessNewSignatureRetryable( + err := agg.ProcessNewSignature( context.Background(), taskIndex, signedTaskResponse.BatchIdentifierHash, &signedTaskResponse.BlsSignature, signedTaskResponse.OperatorId, ) @@ -113,7 +113,7 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // |---RETRYABLE---| -func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { +func (agg *Aggregator) ProcessNewSignature(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { processNewSignature_func := func() error { return agg.blsAggregationService.ProcessNewSignature( ctx, taskIndex, taskResponse, @@ -124,7 +124,7 @@ func (agg *Aggregator) ProcessNewSignatureRetryable(ctx context.Context, taskInd return retry.Retry(processNewSignature_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (agg *Aggregator) GetTaskIndexRetryable(batchIdentifierHash [32]byte) (uint32, error) { +func (agg *Aggregator) GetTaskIndex(batchIdentifierHash [32]byte) (uint32, error) { getTaskIndex_func := func() (uint32, error) { agg.taskMutex.Lock() agg.AggregatorConfig.BaseConfig.Logger.Info("- Locked Resources: Starting processing of Response") diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 9aada4a27..43bfa2a88 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -66,13 +66,13 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) // Subscribe to new tasks - sub, err := SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err := SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { s.logger.Error("Primary failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err } - subFallback, err := SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err := SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { s.logger.Error("Fallback failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err @@ -114,14 +114,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err = SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err = SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { errorChannel <- err } @@ -137,13 +137,13 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) // Subscribe to new tasks - sub, err := SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err := SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { s.logger.Error("Primary failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err } - subFallback, err := SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err := SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { s.logger.Error("Fallback failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err @@ -185,14 +185,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err = SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err = SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { errorChannel <- err } @@ -258,7 +258,7 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { - latestBlock, err := s.BlockNumberRetryable(context.Background()) + latestBlock, err := s.BlockNumber(context.Background()) if err != nil { return nil, err } @@ -271,7 +271,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag fromBlock = latestBlock - BlockInterval } - logs, err := s.FilterBatchV2Retryable(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + logs, err := s.FilterBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) if err != nil { return nil, err } @@ -293,7 +293,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) + state, err := s.BatchesState(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -307,7 +307,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { - latestBlock, err := s.BlockNumberRetryable(context.Background()) + latestBlock, err := s.BlockNumber(context.Background()) if err != nil { return nil, err } @@ -320,7 +320,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag fromBlock = latestBlock - BlockInterval } - logs, err := s.FilterBatchV3Retryable(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + logs, err := s.FilterBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) if err != nil { return nil, err } @@ -342,7 +342,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) + state, err := s.BatchesState(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -355,7 +355,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag } func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { - currentBlock, err := s.BlockNumberRetryable(context.Background()) + currentBlock, err := s.BlockNumber(context.Background()) if err != nil { return err } @@ -363,7 +363,7 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { if currentBlock <= startBlock { // should really be == but just in case // Subscribe to new head c := make(chan *types.Header) - sub, err := s.SubscribeNewHeadRetryable(context.Background(), c) + sub, err := s.SubscribeNewHead(context.Background(), c) if err != nil { return err } diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index c887c1c0d..41f961f66 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -73,7 +73,7 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*common.Hash, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction - tx, err := w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err := w.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // Send the transaction txOpts.NoSend = false txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit - tx, err = w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err = w.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err } @@ -102,7 +102,7 @@ func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bin w.logger.Info("Simulated cost", "cost", simulatedCost) // Get RespondToTaskFeeLimit - batchState, err := w.BatchesStateRetryable(&bind.CallOpts{}, batchIdentifierHash) + batchState, err := w.BatchesState(&bind.CallOpts{}, batchIdentifierHash) if err != nil { // Fallback also failed // Proceed to check values against simulated costs @@ -136,7 +136,7 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - aggregatorBalance, err := w.BalanceAtRetryable(ctx, aggregatorAddress, nil) + aggregatorBalance, err := w.BalanceAt(ctx, aggregatorAddress, nil) if err != nil { // Ignore and continue. w.logger.Error("failed to get aggregator balance: %v", err) @@ -151,7 +151,7 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byte) error { // Get batcher balance - batcherBalance, err := w.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) + batcherBalance, err := w.BatcherBalances(&bind.CallOpts{}, senderAddress) if err != nil { // Ignore and continue. w.logger.Error("Failed to get batcherBalance", "error", err) diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index c5eca3d3b..84a41863e 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -16,7 +16,7 @@ import ( // |---AVS_WRITER---| -func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { +func (w *AvsWriter) RespondToTaskV2(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { var ( tx *types.Transaction err error @@ -36,7 +36,7 @@ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkl return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { +func (w *AvsWriter) BatchesState(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int @@ -56,7 +56,7 @@ func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (s return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BatcherBalancesRetryable(opts *bind.CallOpts, senderAddress common.Address) (*big.Int, error) { +func (w *AvsWriter) BatcherBalances(opts *bind.CallOpts, senderAddress common.Address) (*big.Int, error) { batcherBalances_func := func() (*big.Int, error) { batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(opts, senderAddress) if err != nil { @@ -67,7 +67,7 @@ func (w *AvsWriter) BatcherBalancesRetryable(opts *bind.CallOpts, senderAddress return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { +func (w *AvsWriter) BalanceAt(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { balanceAt_func := func() (*big.Int, error) { aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) if err != nil { @@ -80,7 +80,7 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co // |---AVS_SUBSCRIBER---| -func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { +func (s *AvsSubscriber) BlockNumber(ctx context.Context) (uint64, error) { latestBlock_func := func() (uint64, error) { latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) if err != nil { @@ -91,21 +91,21 @@ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) FilterBatchV2Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { +func (s *AvsSubscriber) FilterBatchV2(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(opts, batchMerkleRoot) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) FilterBatchV3Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { +func (s *AvsSubscriber) FilterBatchV3(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(opts, batchMerkleRoot) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { +func (s *AvsSubscriber) BatchesState(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int @@ -121,7 +121,7 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte return retry.RetryWithData(batchState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { +func (s *AvsSubscriber) SubscribeNewHead(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { subscribeNewHead_func := func() (ethereum.Subscription, error) { sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) if err != nil { @@ -132,7 +132,7 @@ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func SubscribeToNewTasksV2Retrayable( +func SubscribeToNewTasksV2( opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, @@ -144,7 +144,7 @@ func SubscribeToNewTasksV2Retrayable( return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func SubscribeToNewTasksV3Retryable( +func SubscribeToNewTasksV3( opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, diff --git a/core/retry_test.go b/core/retry_test.go index c9883fcbe..e55f67326 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -132,7 +132,7 @@ func TestAnvilSetupKill(t *testing.T) { // |--Aggreagator Retry Tests--| -func TestWaitForTransactionReceiptRetryable(t *testing.T) { +func TestWaitForTransactionReceipt(t *testing.T) { to := common.BytesToAddress([]byte{0x11}) tx := types.NewTx(&types.AccessListTx{ @@ -152,7 +152,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + _, err = utils.WaitForTransactionReceipt(*client, context.Background(), hash) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) if !strings.Contains(err.Error(), "not found") { t.Errorf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) @@ -164,7 +164,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { return } - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + _, err = utils.WaitForTransactionReceipt(*client, context.Background(), hash) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) @@ -180,7 +180,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash) + _, err = utils.WaitForTransactionReceipt(*client, context.Background(), hash) assert.NotNil(t, err) if !strings.Contains(err.Error(), "not found") { t.Errorf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) @@ -197,7 +197,7 @@ func TestWaitForTransactionReceiptRetryable(t *testing.T) { // The originates within the eigen-sdk and as of 8/11/24 is currently working to be fixed. /* -func TestInitializeNewTaskRetryable(t *testing.T) { +func TestInitializeNewTask(t *testing.T) { _, _, err := SetupAnvil(8545) if err != nil { @@ -213,7 +213,7 @@ func TestInitializeNewTaskRetryable(t *testing.T) { quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(byte(0))} quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(byte(57))} - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + err = agg.InitializeNewTask(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -221,7 +221,7 @@ func TestInitializeNewTaskRetryable(t *testing.T) { return } - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + err = agg.InitializeNewTask(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) assert.NotNil(t, err) t.Errorf("Error setting Avs Subscriber: %s\n", err) @@ -230,7 +230,7 @@ func TestInitializeNewTaskRetryable(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - err = agg.InitializeNewTaskRetryable(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) + err = agg.InitializeNewTask(0, 1, quorumNums, quorumThresholdPercentages, 1*time.Second) assert.Nil(t, err) t.Errorf("Error setting Avs Subscriber: %s\n", err) @@ -242,7 +242,7 @@ func TestInitializeNewTaskRetryable(t *testing.T) { */ /* -func TestGetTaskIndexRetryable(t *testing.T) { +func TestGetTaskIndex(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { @@ -258,7 +258,7 @@ func TestGetTaskIndexRetryable(t *testing.T) { zero_bytes := [32]byte{} // Task is not present in map should return transient error - _, err = agg.GetTaskIndexRetryable(zero_bytes) + _, err = agg.GetTaskIndex(zero_bytes) assert.NotNil(t, err) if !strings.Contains(err.Error(), "Task not found in the internal map") { t.Errorf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) @@ -274,7 +274,7 @@ func TestGetTaskIndexRetryable(t *testing.T) { /* // |--Server Retry Tests--| -func TestProcessNewSignatureRetryable(t *testing.T) { +func TestProcessNewSignature(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { t.Errorf("Error setting up Anvil: %s\n", err) @@ -290,7 +290,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { zero_sig := bls.NewZeroSignature() eigen_bytes := eigentypes.Bytes32{} - err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + err = agg.ProcessNewSignature(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.NotNil(t, err) // Kill Anvil at end of test @@ -299,7 +299,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { return } - err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + err = agg.ProcessNewSignature(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.NotNil(t, err) t.Errorf("Error Processing New Signature: %s\n", err) @@ -308,7 +308,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - err = agg.ProcessNewSignatureRetryable(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) + err = agg.ProcessNewSignature(context.Background(), 0, zero_bytes, zero_sig, eigen_bytes) assert.Nil(t, err) t.Errorf("Error Processing New Signature: %s\n", err) @@ -321,7 +321,7 @@ func TestProcessNewSignatureRetryable(t *testing.T) { // |--AVS-Subscriber Retry Tests--| -func TestSubscribeToNewTasksV3Retryable(t *testing.T) { +func TestSubscribeToNewTasksV3(t *testing.T) { cmd, _, err := SetupAnvil(8545) if err != nil { t.Errorf("Error setting up Anvil: %s\n", err) @@ -338,7 +338,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { t.Errorf("Error setting up Avs Service Bindings: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV3(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -346,7 +346,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { return } - _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV3(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) @@ -362,7 +362,7 @@ func TestSubscribeToNewTasksV3Retryable(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV3(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -388,7 +388,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { t.Errorf("Error setting up Avs Service Bindings: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV2(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -396,7 +396,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { return } - _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV2(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) @@ -412,7 +412,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV2Retrayable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV2(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -433,7 +433,7 @@ func TestBlockNumber(t *testing.T) { if err != nil { return } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = sub.BlockNumber(context.Background()) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -441,7 +441,7 @@ func TestBlockNumber(t *testing.T) { return } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = sub.BlockNumber(context.Background()) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BlockNumber Emitted non Transient error: %s\n", err) @@ -457,7 +457,7 @@ func TestBlockNumber(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumberRetryable(context.Background()) + _, err = sub.BlockNumber(context.Background()) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -477,7 +477,7 @@ func TestFilterBatchV2(t *testing.T) { if err != nil { return } - _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV2(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -485,7 +485,7 @@ func TestFilterBatchV2(t *testing.T) { return } - _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV2(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("FilterBatchV2 Emitted non Transient error: %s\n", err) @@ -501,7 +501,7 @@ func TestFilterBatchV2(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV2(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -521,7 +521,7 @@ func TestFilterBatchV3(t *testing.T) { if err != nil { return } - _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV3(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -529,7 +529,7 @@ func TestFilterBatchV3(t *testing.T) { return } - _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV3(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("FilerBatchV3 Emitted non Transient error: %s\n", err) @@ -545,7 +545,7 @@ func TestFilterBatchV3(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV3(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -567,7 +567,7 @@ func TestBatchesStateSubscriber(t *testing.T) { } zero_bytes := [32]byte{} - _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) + _, err = avsSubscriber.BatchesState(nil, zero_bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -575,7 +575,7 @@ func TestBatchesStateSubscriber(t *testing.T) { return } - _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) + _, err = avsSubscriber.BatchesState(nil, zero_bytes) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) @@ -591,7 +591,7 @@ func TestBatchesStateSubscriber(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) + _, err = avsSubscriber.BatchesState(nil, zero_bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -613,7 +613,7 @@ func TestSubscribeNewHead(t *testing.T) { return } - _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) + _, err = avsSubscriber.SubscribeNewHead(context.Background(), c) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -621,7 +621,7 @@ func TestSubscribeNewHead(t *testing.T) { return } - _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) + _, err = avsSubscriber.SubscribeNewHead(context.Background(), c) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("SubscribeNewHead Emitted non Transient error: %s\n", err) @@ -637,7 +637,7 @@ func TestSubscribeNewHead(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) + _, err = avsSubscriber.SubscribeNewHead(context.Background(), c) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -690,7 +690,7 @@ func TestRespondToTaskV2(t *testing.T) { zero_bytes := [32]byte{} // NOTE: With zero bytes the tx reverts - _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + _, err = w.RespondToTaskV2(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) if !strings.Contains(err.Error(), "execution reverted") { t.Errorf("RespondToTaskV2 did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") @@ -700,7 +700,7 @@ func TestRespondToTaskV2(t *testing.T) { t.Errorf("Error killing process: %v\n", err) } - _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + _, err = w.RespondToTaskV2(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) if _, ok := err.(*backoff.PermanentError); ok { t.Errorf("RespondToTaskV2 Emitted non-Transient error: %s\n", err) @@ -715,7 +715,7 @@ func TestRespondToTaskV2(t *testing.T) { } // NOTE: With zero bytes the tx reverts - _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + _, err = w.RespondToTaskV2(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) if !strings.Contains(err.Error(), "execution reverted") { t.Errorf("RespondToTaskV2 did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") @@ -743,7 +743,7 @@ func TestBatchesStateWriter(t *testing.T) { var bytes [32]byte num.FillBytes(bytes[:]) - _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) + _, err = avsWriter.BatchesState(&bind.CallOpts{}, bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -751,7 +751,7 @@ func TestBatchesStateWriter(t *testing.T) { return } - _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) + _, err = avsWriter.BatchesState(&bind.CallOpts{}, bytes) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BatchesStateWriter Emitted non-Transient error: %s\n", err) @@ -767,7 +767,7 @@ func TestBatchesStateWriter(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) + _, err = avsWriter.BatchesState(&bind.CallOpts{}, bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -790,7 +790,7 @@ func TestBalanceAt(t *testing.T) { aggregator_address := common.HexToAddress("0x0") blockHeight := big.NewInt(13) - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) + _, err = avsWriter.BalanceAt(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -798,7 +798,7 @@ func TestBalanceAt(t *testing.T) { return } - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) + _, err = avsWriter.BalanceAt(context.Background(), aggregator_address, blockHeight) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BalanceAt Emitted non-Transient error: %s\n", err) @@ -814,7 +814,7 @@ func TestBalanceAt(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) + _, err = avsWriter.BalanceAt(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -836,7 +836,7 @@ func TestBatchersBalances(t *testing.T) { } senderAddress := common.HexToAddress("0x0") - _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) + _, err = avsWriter.BatcherBalances(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -844,7 +844,7 @@ func TestBatchersBalances(t *testing.T) { return } - _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) + _, err = avsWriter.BatcherBalances(&bind.CallOpts{}, senderAddress) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BatchersBalances Emitted non-Transient error: %s\n", err) @@ -860,7 +860,7 @@ func TestBatchersBalances(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) + _, err = avsWriter.BatcherBalances(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 8679edbc9..0c9e1a168 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -10,7 +10,7 @@ import ( retry "github.com/yetanotherco/aligned_layer/core" ) -func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { +func WaitForTransactionReceipt(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash) (*types.Receipt, error) { receipt_func := func() (*types.Receipt, error) { return client.TransactionReceipt(ctx, txHash) } From b30d1ced3e64c0c69d672e8b3f3013d166179e31 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Fri, 8 Nov 2024 16:41:46 -0300 Subject: [PATCH 114/135] rename InitializeNewTaskRetryable to InitializeNewTask --- aggregator/pkg/aggregator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index a8fb6c644..50f29617d 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -371,7 +371,7 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by quorumNums := eigentypes.QuorumNums{eigentypes.QuorumNum(QUORUM_NUMBER)} quorumThresholdPercentages := eigentypes.QuorumThresholdPercentages{eigentypes.QuorumThresholdPercentage(QUORUM_THRESHOLD)} - err := agg.InitializeNewTaskRetryable(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) + err := agg.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, 100*time.Second) // FIXME(marian): When this errors, should we retry initializing new task? Logging fatal for now. if err != nil { agg.logger.Fatalf("BLS aggregation service error when initializing new task: %s", err) @@ -385,7 +385,7 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by // |---RETRYABLE---| -func (agg *Aggregator) InitializeNewTaskRetryable(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { +func (agg *Aggregator) InitializeNewTask(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { initilizeNewTask_func := func() error { return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, timeToExpiry) } From 0345248aaedc29495173b4553d44befe02eeae60 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Sat, 9 Nov 2024 02:10:37 -0300 Subject: [PATCH 115/135] add Retryable back to function names --- core/chainio/avs_subscriber.go | 32 ++++++++--------- core/chainio/avs_writer.go | 10 +++--- core/chainio/retryable.go | 22 ++++++------ core/retry_test.go | 66 +++++++++++++++++----------------- 4 files changed, 65 insertions(+), 65 deletions(-) diff --git a/core/chainio/avs_subscriber.go b/core/chainio/avs_subscriber.go index 43bfa2a88..ea810da62 100644 --- a/core/chainio/avs_subscriber.go +++ b/core/chainio/avs_subscriber.go @@ -66,13 +66,13 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2) // Subscribe to new tasks - sub, err := SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err := SubscribeToNewTasksV2Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { s.logger.Error("Primary failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err } - subFallback, err := SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err := SubscribeToNewTasksV2Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { s.logger.Error("Fallback failed to subscribe to new AlignedLayer V2 tasks after %d retries", retry.NumRetries, "err", err) return nil, err @@ -114,14 +114,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV2(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err = SubscribeToNewTasksV2Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = SubscribeToNewTasksV2(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err = SubscribeToNewTasksV2Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { errorChannel <- err } @@ -137,13 +137,13 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema internalChannel := make(chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3) // Subscribe to new tasks - sub, err := SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err := SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { s.logger.Error("Primary failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err } - subFallback, err := SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err := SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { s.logger.Error("Fallback failed to subscribe to new AlignedLayer V3 tasks after %d retries", MaxRetries, "err", err) return nil, err @@ -185,14 +185,14 @@ func (s *AvsSubscriber) SubscribeToNewTasksV3(newTaskCreatedChan chan *servicema case err := <-sub.Err(): s.logger.Warn("Error in new task subscription", "err", err) sub.Unsubscribe() - sub, err = SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) + sub, err = SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManager, internalChannel, nil) if err != nil { errorChannel <- err } case err := <-subFallback.Err(): s.logger.Warn("Error in fallback new task subscription", "err", err) subFallback.Unsubscribe() - subFallback, err = SubscribeToNewTasksV3(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) + subFallback, err = SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.AvsContractBindings.ServiceManagerFallback, internalChannel, nil) if err != nil { errorChannel <- err } @@ -258,7 +258,7 @@ func (s *AvsSubscriber) processNewBatchV3(batch *servicemanager.ContractAlignedL // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, error) { - latestBlock, err := s.BlockNumber(context.Background()) + latestBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return nil, err } @@ -271,7 +271,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag fromBlock = latestBlock - BlockInterval } - logs, err := s.FilterBatchV2(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + logs, err := s.FilterBatchV2Retryable(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) if err != nil { return nil, err } @@ -293,7 +293,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - state, err := s.BatchesState(nil, batchIdentifierHash) + state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -307,7 +307,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV2() (*servicemanag // getLatestNotRespondedTaskFromEthereum queries the blockchain for the latest not responded task using the FilterNewBatch method. func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, error) { - latestBlock, err := s.BlockNumber(context.Background()) + latestBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return nil, err } @@ -320,7 +320,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag fromBlock = latestBlock - BlockInterval } - logs, err := s.FilterBatchV3(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) + logs, err := s.FilterBatchV3Retryable(&bind.FilterOpts{Start: fromBlock, End: nil, Context: context.Background()}, nil) if err != nil { return nil, err } @@ -342,7 +342,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag batchIdentifier := append(lastLog.BatchMerkleRoot[:], lastLog.SenderAddress[:]...) batchIdentifierHash := *(*[32]byte)(crypto.Keccak256(batchIdentifier)) - state, err := s.BatchesState(nil, batchIdentifierHash) + state, err := s.BatchesStateRetryable(nil, batchIdentifierHash) if err != nil { return nil, err } @@ -355,7 +355,7 @@ func (s *AvsSubscriber) getLatestNotRespondedTaskFromEthereumV3() (*servicemanag } func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { - currentBlock, err := s.BlockNumber(context.Background()) + currentBlock, err := s.BlockNumberRetryable(context.Background()) if err != nil { return err } @@ -363,7 +363,7 @@ func (s *AvsSubscriber) WaitForOneBlock(startBlock uint64) error { if currentBlock <= startBlock { // should really be == but just in case // Subscribe to new head c := make(chan *types.Header) - sub, err := s.SubscribeNewHead(context.Background(), c) + sub, err := s.SubscribeNewHeadRetryable(context.Background(), c) if err != nil { return err } diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 41f961f66..c887c1c0d 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -73,7 +73,7 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*common.Hash, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction - tx, err := w.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err := w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // Send the transaction txOpts.NoSend = false txOpts.GasLimit = tx.Gas() * 110 / 100 // Add 10% to the gas limit - tx, err = w.RespondToTaskV2(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + tx, err = w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { return nil, err } @@ -102,7 +102,7 @@ func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bin w.logger.Info("Simulated cost", "cost", simulatedCost) // Get RespondToTaskFeeLimit - batchState, err := w.BatchesState(&bind.CallOpts{}, batchIdentifierHash) + batchState, err := w.BatchesStateRetryable(&bind.CallOpts{}, batchIdentifierHash) if err != nil { // Fallback also failed // Proceed to check values against simulated costs @@ -136,7 +136,7 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - aggregatorBalance, err := w.BalanceAt(ctx, aggregatorAddress, nil) + aggregatorBalance, err := w.BalanceAtRetryable(ctx, aggregatorAddress, nil) if err != nil { // Ignore and continue. w.logger.Error("failed to get aggregator balance: %v", err) @@ -151,7 +151,7 @@ func (w *AvsWriter) compareAggregatorBalance(amount *big.Int, aggregatorAddress func (w *AvsWriter) compareBatcherBalance(amount *big.Int, senderAddress [20]byte) error { // Get batcher balance - batcherBalance, err := w.BatcherBalances(&bind.CallOpts{}, senderAddress) + batcherBalance, err := w.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) if err != nil { // Ignore and continue. w.logger.Error("Failed to get batcherBalance", "error", err) diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index 84a41863e..059bcf109 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -16,7 +16,7 @@ import ( // |---AVS_WRITER---| -func (w *AvsWriter) RespondToTaskV2(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { +func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { var ( tx *types.Transaction err error @@ -36,7 +36,7 @@ func (w *AvsWriter) RespondToTaskV2(opts *bind.TransactOpts, batchMerkleRoot [32 return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BatchesState(opts *bind.CallOpts, arg0 [32]byte) (struct { +func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int @@ -56,7 +56,7 @@ func (w *AvsWriter) BatchesState(opts *bind.CallOpts, arg0 [32]byte) (struct { return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BatcherBalances(opts *bind.CallOpts, senderAddress common.Address) (*big.Int, error) { +func (w *AvsWriter) BatcherBalancesRetryable(opts *bind.CallOpts, senderAddress common.Address) (*big.Int, error) { batcherBalances_func := func() (*big.Int, error) { batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(opts, senderAddress) if err != nil { @@ -67,7 +67,7 @@ func (w *AvsWriter) BatcherBalances(opts *bind.CallOpts, senderAddress common.Ad return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (w *AvsWriter) BalanceAt(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { +func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { balanceAt_func := func() (*big.Int, error) { aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) if err != nil { @@ -80,7 +80,7 @@ func (w *AvsWriter) BalanceAt(ctx context.Context, aggregatorAddress common.Addr // |---AVS_SUBSCRIBER---| -func (s *AvsSubscriber) BlockNumber(ctx context.Context) (uint64, error) { +func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { latestBlock_func := func() (uint64, error) { latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) if err != nil { @@ -91,21 +91,21 @@ func (s *AvsSubscriber) BlockNumber(ctx context.Context) (uint64, error) { return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) FilterBatchV2(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { +func (s *AvsSubscriber) FilterBatchV2Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(opts, batchMerkleRoot) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) FilterBatchV3(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { +func (s *AvsSubscriber) FilterBatchV3Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(opts, batchMerkleRoot) } return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) BatchesState(opts *bind.CallOpts, arg0 [32]byte) (struct { +func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool RespondToTaskFeeLimit *big.Int @@ -121,7 +121,7 @@ func (s *AvsSubscriber) BatchesState(opts *bind.CallOpts, arg0 [32]byte) (struct return retry.RetryWithData(batchState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func (s *AvsSubscriber) SubscribeNewHead(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { +func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { subscribeNewHead_func := func() (ethereum.Subscription, error) { sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) if err != nil { @@ -132,7 +132,7 @@ func (s *AvsSubscriber) SubscribeNewHead(ctx context.Context, c chan<- *types.He return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func SubscribeToNewTasksV2( +func SubscribeToNewTasksV2Retryable( opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV2, @@ -144,7 +144,7 @@ func SubscribeToNewTasksV2( return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } -func SubscribeToNewTasksV3( +func SubscribeToNewTasksV3Retryable( opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, newTaskCreatedChan chan *servicemanager.ContractAlignedLayerServiceManagerNewBatchV3, diff --git a/core/retry_test.go b/core/retry_test.go index e55f67326..9c5349829 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -338,7 +338,7 @@ func TestSubscribeToNewTasksV3(t *testing.T) { t.Errorf("Error setting up Avs Service Bindings: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV3(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -346,7 +346,7 @@ func TestSubscribeToNewTasksV3(t *testing.T) { return } - _, err = chainio.SubscribeToNewTasksV3(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("SubscribeToNewTasksV3 Emitted non Transient error: %s\n", err) @@ -362,7 +362,7 @@ func TestSubscribeToNewTasksV3(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV3(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV3Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -388,7 +388,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { t.Errorf("Error setting up Avs Service Bindings: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV2(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV2Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -396,7 +396,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { return } - _, err = chainio.SubscribeToNewTasksV2(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV2Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("SubscribeToNewTasksV2 Emitted non Transient error: %s\n", err) @@ -412,7 +412,7 @@ func TestSubscribeToNewTasksV2(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = chainio.SubscribeToNewTasksV2(&bind.WatchOpts{}, s.ServiceManager, channel, nil) + _, err = chainio.SubscribeToNewTasksV2Retryable(&bind.WatchOpts{}, s.ServiceManager, channel, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -433,7 +433,7 @@ func TestBlockNumber(t *testing.T) { if err != nil { return } - _, err = sub.BlockNumber(context.Background()) + _, err = sub.BlockNumberRetryable(context.Background()) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -441,7 +441,7 @@ func TestBlockNumber(t *testing.T) { return } - _, err = sub.BlockNumber(context.Background()) + _, err = sub.BlockNumberRetryable(context.Background()) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BlockNumber Emitted non Transient error: %s\n", err) @@ -457,7 +457,7 @@ func TestBlockNumber(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = sub.BlockNumber(context.Background()) + _, err = sub.BlockNumberRetryable(context.Background()) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -477,7 +477,7 @@ func TestFilterBatchV2(t *testing.T) { if err != nil { return } - _, err = avsSubscriber.FilterBatchV2(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -485,7 +485,7 @@ func TestFilterBatchV2(t *testing.T) { return } - _, err = avsSubscriber.FilterBatchV2(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("FilterBatchV2 Emitted non Transient error: %s\n", err) @@ -501,7 +501,7 @@ func TestFilterBatchV2(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.FilterBatchV2(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV2Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -521,7 +521,7 @@ func TestFilterBatchV3(t *testing.T) { if err != nil { return } - _, err = avsSubscriber.FilterBatchV3(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -529,7 +529,7 @@ func TestFilterBatchV3(t *testing.T) { return } - _, err = avsSubscriber.FilterBatchV3(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("FilerBatchV3 Emitted non Transient error: %s\n", err) @@ -545,7 +545,7 @@ func TestFilterBatchV3(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.FilterBatchV3(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) + _, err = avsSubscriber.FilterBatchV3Retryable(&bind.FilterOpts{Start: 0, End: nil, Context: context.Background()}, nil) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -567,7 +567,7 @@ func TestBatchesStateSubscriber(t *testing.T) { } zero_bytes := [32]byte{} - _, err = avsSubscriber.BatchesState(nil, zero_bytes) + _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -575,7 +575,7 @@ func TestBatchesStateSubscriber(t *testing.T) { return } - _, err = avsSubscriber.BatchesState(nil, zero_bytes) + _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BatchesStateSubscriber Emitted non Transient error: %s\n", err) @@ -591,7 +591,7 @@ func TestBatchesStateSubscriber(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.BatchesState(nil, zero_bytes) + _, err = avsSubscriber.BatchesStateRetryable(nil, zero_bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -613,7 +613,7 @@ func TestSubscribeNewHead(t *testing.T) { return } - _, err = avsSubscriber.SubscribeNewHead(context.Background(), c) + _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -621,7 +621,7 @@ func TestSubscribeNewHead(t *testing.T) { return } - _, err = avsSubscriber.SubscribeNewHead(context.Background(), c) + _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("SubscribeNewHead Emitted non Transient error: %s\n", err) @@ -637,7 +637,7 @@ func TestSubscribeNewHead(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsSubscriber.SubscribeNewHead(context.Background(), c) + _, err = avsSubscriber.SubscribeNewHeadRetryable(context.Background(), c) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -690,7 +690,7 @@ func TestRespondToTaskV2(t *testing.T) { zero_bytes := [32]byte{} // NOTE: With zero bytes the tx reverts - _, err = w.RespondToTaskV2(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) if !strings.Contains(err.Error(), "execution reverted") { t.Errorf("RespondToTaskV2 did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") @@ -700,7 +700,7 @@ func TestRespondToTaskV2(t *testing.T) { t.Errorf("Error killing process: %v\n", err) } - _, err = w.RespondToTaskV2(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) if _, ok := err.(*backoff.PermanentError); ok { t.Errorf("RespondToTaskV2 Emitted non-Transient error: %s\n", err) @@ -715,7 +715,7 @@ func TestRespondToTaskV2(t *testing.T) { } // NOTE: With zero bytes the tx reverts - _, err = w.RespondToTaskV2(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) + _, err = w.RespondToTaskV2Retryable(&txOpts, zero_bytes, aggregator_address, nonSignerStakesAndSignature) assert.NotNil(t, err) if !strings.Contains(err.Error(), "execution reverted") { t.Errorf("RespondToTaskV2 did not emit the expected message: %q doesn't contain %q", err.Error(), "execution reverted: custom error 0x2396d34e:") @@ -743,7 +743,7 @@ func TestBatchesStateWriter(t *testing.T) { var bytes [32]byte num.FillBytes(bytes[:]) - _, err = avsWriter.BatchesState(&bind.CallOpts{}, bytes) + _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -751,7 +751,7 @@ func TestBatchesStateWriter(t *testing.T) { return } - _, err = avsWriter.BatchesState(&bind.CallOpts{}, bytes) + _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BatchesStateWriter Emitted non-Transient error: %s\n", err) @@ -767,7 +767,7 @@ func TestBatchesStateWriter(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BatchesState(&bind.CallOpts{}, bytes) + _, err = avsWriter.BatchesStateRetryable(&bind.CallOpts{}, bytes) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -790,7 +790,7 @@ func TestBalanceAt(t *testing.T) { aggregator_address := common.HexToAddress("0x0") blockHeight := big.NewInt(13) - _, err = avsWriter.BalanceAt(context.Background(), aggregator_address, blockHeight) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -798,7 +798,7 @@ func TestBalanceAt(t *testing.T) { return } - _, err = avsWriter.BalanceAt(context.Background(), aggregator_address, blockHeight) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BalanceAt Emitted non-Transient error: %s\n", err) @@ -814,7 +814,7 @@ func TestBalanceAt(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BalanceAt(context.Background(), aggregator_address, blockHeight) + _, err = avsWriter.BalanceAtRetryable(context.Background(), aggregator_address, blockHeight) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -836,7 +836,7 @@ func TestBatchersBalances(t *testing.T) { } senderAddress := common.HexToAddress("0x0") - _, err = avsWriter.BatcherBalances(&bind.CallOpts{}, senderAddress) + _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { @@ -844,7 +844,7 @@ func TestBatchersBalances(t *testing.T) { return } - _, err = avsWriter.BatcherBalances(&bind.CallOpts{}, senderAddress) + _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("BatchersBalances Emitted non-Transient error: %s\n", err) @@ -860,7 +860,7 @@ func TestBatchersBalances(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = avsWriter.BatcherBalances(&bind.CallOpts{}, senderAddress) + _, err = avsWriter.BatcherBalancesRetryable(&bind.CallOpts{}, senderAddress) assert.Nil(t, err) if err := cmd.Process.Kill(); err != nil { From 67f59dd94e0c1d1c752394a21a08ee6ce605ee83 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 11 Nov 2024 10:58:43 -0300 Subject: [PATCH 116/135] refactor: tracing and docs --- core/chainio/avs_writer.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 12dc3b282..418d69b90 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/big" - "strings" "time" "github.com/Layr-Labs/eigensdk-go/chainio/clients" @@ -87,44 +86,44 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { - return nil, retry.PermanentError{Inner: err} + return nil, err } // Set the nonce, as we might have to replace the transaction with a higher gas price txNonce := big.NewInt(int64(tx.Nonce())) txOpts.Nonce = txNonce + txOpts.GasPrice = tx.GasPrice() txOpts.NoSend = false i := 0 respondToTaskV2Func := func() (*types.Receipt, error) { - // We bump only when the timeout for waiting for the receipt has passed - // not when an rpc failed gasPrice, err := w.ClientFallback.SuggestGasPrice(context.Background()) if err != nil { return nil, err } + bumpedGasPrice := utils.CalculateGasPriceBumpBasedOnRetry(gasPrice, gasBumpPercentage, gasBumpIncrementalPercentage, i) // new bumped gas price must be higher than the last one (this should hardly ever happen though) - if gasPrice.Cmp(txOpts.GasPrice) > 0 { + if bumpedGasPrice.Cmp(txOpts.GasPrice) > 0 { txOpts.GasPrice = bumpedGasPrice } else { - // bump the last price 10% to replace it. - bumpAmount := new(big.Int).Mul(txOpts.GasPrice, big.NewInt(10)) - bumpAmount = new(big.Int).Div(bumpAmount, big.NewInt(100)) - txOpts.GasPrice = new(big.Int).Mul(txOpts.GasPrice, bumpAmount) + // bump the last price to replace it. + txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(txOpts.GasPrice, gasBumpIncrementalPercentage, 0, 0) } + if i > 0 { onRetry(txOpts.GasPrice) } err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) if err != nil { - // We bump the fee so much that the transaction cost is more expensive than the batcher fee limit return nil, retry.PermanentError{Inner: err} } + w.logger.Infof("Sending RespondToTask transaction with a gas price of %v", txOpts.GasPrice) + tx, err = w.RespondToTaskV2Retryable(&txOpts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if strings.Contains(err.Error(), "reverted") { + if err != nil { return nil, err } @@ -137,13 +136,14 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe // we increment the i here to add an incremental percentage to increase the odds of being included in the next blocks i++ + w.logger.Infof("RespondToTask receipt waiting timeout has passed, will try again...") if err != nil { return nil, err } return nil, fmt.Errorf("transaction failed") } - return retry.RetryWithData(respondToTaskV2Func, 1000, 2, 0, 60, retry.MaxElapsedTime) + return retry.RetryWithData(respondToTaskV2Func, 1000, 2, 0, 60000, retry.MaxElapsedTime) } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { From 18f77bb6183a252064a18dc18b4436611fb8d279 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 11 Nov 2024 10:59:34 -0300 Subject: [PATCH 117/135] chore: adjust bump percentages --- config-files/config-aggregator.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index f33ac0070..3e2d6461a 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -1,23 +1,23 @@ # Common variables for all the services # 'production' only prints info and above. 'development' also prints debug -environment: "production" -aligned_layer_deployment_config_file_path: "./contracts/script/output/devnet/alignedlayer_deployment_output.json" -eigen_layer_deployment_config_file_path: "./contracts/script/output/devnet/eigenlayer_deployment_output.json" -eth_rpc_url: "http://localhost:8545" -eth_rpc_url_fallback: "http://localhost:8545" -eth_ws_url: "ws://localhost:8545" -eth_ws_url_fallback: "ws://localhost:8545" -eigen_metrics_ip_port_address: "localhost:9090" +environment: 'production' +aligned_layer_deployment_config_file_path: './contracts/script/output/devnet/alignedlayer_deployment_output.json' +eigen_layer_deployment_config_file_path: './contracts/script/output/devnet/eigenlayer_deployment_output.json' +eth_rpc_url: 'http://localhost:8545' +eth_rpc_url_fallback: 'http://localhost:8545' +eth_ws_url: 'ws://localhost:8545' +eth_ws_url_fallback: 'ws://localhost:8545' +eigen_metrics_ip_port_address: 'localhost:9090' ## ECDSA Configurations ecdsa: - private_key_store_path: "config-files/anvil.aggregator.ecdsa.key.json" - private_key_store_password: "" + private_key_store_path: 'config-files/anvil.aggregator.ecdsa.key.json' + private_key_store_password: '' ## BLS Configurations bls: - private_key_store_path: "config-files/anvil.aggregator.bls.key.json" - private_key_store_password: "" + private_key_store_path: 'config-files/anvil.aggregator.bls.key.json' + private_key_store_password: '' # ## Batcher configurations # batcher: # block_interval: 3 @@ -37,7 +37,7 @@ aggregator: garbage_collector_period: 2m #The period of the GC process. Suggested value for Prod: '168h' (7 days) garbage_collector_tasks_age: 20 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days) garbage_collector_tasks_interval: 10 #The interval of queried blocks to get an old batch. Suggested value for prod: '900' (3 hours) - gas_base_bump_percentage: 20 # How much to bump gas price when responding to task. Suggested value 20% + gas_base_bump_percentage: 10 # How much to bump gas price when responding to task. Suggested value 20% gas_bump_incremental_percentage: 5 # An extra percentage to bump every retry i*5 when responding to task. Suggested value 5% time_to_wait_before_bump: 36s # The time to wait for the receipt when responding to task. Suggested value 36 seconds (3 blocks) From 509f6efff200d4f29cb8f13d0866ea7e2b3dad29 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 11 Nov 2024 11:17:08 -0300 Subject: [PATCH 118/135] fix: telemetry bump gas price endpoint --- .../controllers/trace_controller.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex b/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex index 210e4b667..b9e334652 100644 --- a/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex +++ b/telemetry_api/lib/telemetry_api_web/controllers/trace_controller.ex @@ -3,7 +3,7 @@ defmodule TelemetryApiWeb.TraceController do alias TelemetryApi.Traces - action_fallback TelemetryApiWeb.FallbackController + action_fallback(TelemetryApiWeb.FallbackController) @doc """ Create a trace for a NewTask with the given merkle_root @@ -121,15 +121,15 @@ defmodule TelemetryApiWeb.TraceController do |> render(:show_merkle, merkle_root: merkle_root) end end - + @doc """ Registers a gas price bump in the trace of the given merkle_root Method: POST aggregatorTaskGasPriceBump """ - def aggregator_task_gas_price_bump(conn, %{ - "merkle_root" => merkle_root, - "bumped_gas_price" => bumped_gas_price - }) do + def aggregator_task_gas_price_bumped(conn, %{ + "merkle_root" => merkle_root, + "bumped_gas_price" => bumped_gas_price + }) do with :ok <- Traces.aggregator_task_gas_price_bumped(merkle_root, bumped_gas_price) do conn |> put_status(:ok) From daa8f400f371a67b43120a692f9161e7edab3907 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 11 Nov 2024 12:26:22 -0300 Subject: [PATCH 119/135] refactor: get gas price retryable --- core/chainio/avs_writer.go | 2 +- core/utils/eth_client_utils.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 418d69b90..9eba6696e 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -97,7 +97,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe i := 0 respondToTaskV2Func := func() (*types.Receipt, error) { - gasPrice, err := w.ClientFallback.SuggestGasPrice(context.Background()) + gasPrice, err := utils.GetGasPriceRetryable(w.Client, context.Background()) if err != nil { return nil, err } diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 7434c5513..2ec8aa6c1 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -57,3 +57,18 @@ func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercent return bumpedGasPrice } + +func GetGasPriceRetryable(client eth.InstrumentedClient, ctx context.Context) (*big.Int, error) { + respondToTaskV2_func := func() (*big.Int, error) { + gasPrice, err := client.SuggestGasPrice(context.Background()) + if err != nil { + gasPrice, err = client.SuggestGasPrice(context.Background()) + if err != nil { + return nil, err + } + } + + return gasPrice, nil + } + return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) +} From 6051a57341ef76bafb366292ce06e123bc47bc6b Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 11 Nov 2024 12:29:43 -0300 Subject: [PATCH 120/135] refactor: address review comments --- aggregator/pkg/aggregator.go | 4 ++-- core/chainio/avs_writer.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 34e62b961..d3b5d1542 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -302,11 +302,11 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "senderAddress", hex.EncodeToString(senderAddress[:]), "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) - onRetry := func(bumpedGasPrice *big.Int) { + onGasPriceBumped := func(bumpedGasPrice *big.Int) { agg.metrics.IncBumpedGasPriceForAggregatedResponse() agg.telemetry.BumpedTaskGasPrice(batchMerkleRoot, bumpedGasPrice.String()) } - receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature, agg.AggregatorConfig.Aggregator.GasBaseBumpPercentage, agg.AggregatorConfig.Aggregator.GasBumpIncrementalPercentage, agg.AggregatorConfig.Aggregator.TimeToWaitBeforeBump, onRetry) + receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature, agg.AggregatorConfig.Aggregator.GasBaseBumpPercentage, agg.AggregatorConfig.Aggregator.GasBumpIncrementalPercentage, agg.AggregatorConfig.Aggregator.TimeToWaitBeforeBump, onGasPriceBumped) if err != nil { agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Error sending aggregated response for batch %s. Error: %s", hex.EncodeToString(batchIdentifierHash[:]), err) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 9eba6696e..b62de11a0 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -75,7 +75,7 @@ func NewAvsWriterFromConfig(baseConfig *config.BaseConfig, ecdsaConfig *config.E // Sends AggregatedResponse and waits for the receipt for three blocks, if not received // it will try again bumping the last tx gas price based on `CalculateGasPriceBump` // This process happens indefinitely until the transaction is included. -func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, gasBumpPercentage uint, gasBumpIncrementalPercentage uint, timeToWaitBeforeBump time.Duration, onRetry func(*big.Int)) (*types.Receipt, error) { +func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMerkleRoot [32]byte, senderAddress [20]byte, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature, gasBumpPercentage uint, gasBumpIncrementalPercentage uint, timeToWaitBeforeBump time.Duration, onGasPriceBumped func(*big.Int)) (*types.Receipt, error) { txOpts := *w.Signer.GetTxOpts() txOpts.NoSend = true // simulate the transaction @@ -107,12 +107,12 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe if bumpedGasPrice.Cmp(txOpts.GasPrice) > 0 { txOpts.GasPrice = bumpedGasPrice } else { - // bump the last price to replace it. + // bump the last tx gas price a little by `gasBumpIncrementalPercentage` to replace it. txOpts.GasPrice = utils.CalculateGasPriceBumpBasedOnRetry(txOpts.GasPrice, gasBumpIncrementalPercentage, 0, 0) } if i > 0 { - onRetry(txOpts.GasPrice) + onGasPriceBumped(txOpts.GasPrice) } err = w.checkRespondToTaskFeeLimit(tx, txOpts, batchIdentifierHash, senderAddress) From 5adf0996a343983078cbba13c025cc02bbc90b00 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Mon, 11 Nov 2024 12:31:49 -0300 Subject: [PATCH 121/135] chore: config-aggregator comments --- config-files/config-aggregator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 3e2d6461a..29ee82ba7 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -37,8 +37,8 @@ aggregator: garbage_collector_period: 2m #The period of the GC process. Suggested value for Prod: '168h' (7 days) garbage_collector_tasks_age: 20 #The age of tasks that will be removed by the GC, in blocks. Suggested value for prod: '216000' (30 days) garbage_collector_tasks_interval: 10 #The interval of queried blocks to get an old batch. Suggested value for prod: '900' (3 hours) - gas_base_bump_percentage: 10 # How much to bump gas price when responding to task. Suggested value 20% - gas_bump_incremental_percentage: 5 # An extra percentage to bump every retry i*5 when responding to task. Suggested value 5% + gas_base_bump_percentage: 10 # How much to bump gas price when responding to task. Suggested value 10% + gas_bump_incremental_percentage: 5 # An extra percentage to bump every retry i*5 when responding to task. Suggested value 2% time_to_wait_before_bump: 36s # The time to wait for the receipt when responding to task. Suggested value 36 seconds (3 blocks) ## Operator Configurations From ba7e75d9553a736955e9efe3d78bdc60d9062a30 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 11 Nov 2024 15:16:57 -0300 Subject: [PATCH 122/135] comments + adjust retry params for contract calls + add permanent errors --- aggregator/pkg/aggregator.go | 18 ++++++++- aggregator/pkg/server.go | 24 ++++++++++- core/chainio/retryable.go | 77 +++++++++++++++++++++++++++++------- core/retry.go | 74 +++++++++++++++++++++++++++++----- 4 files changed, 164 insertions(+), 29 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 50f29617d..b1d78210a 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "strings" "sync" "time" @@ -385,11 +386,24 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by // |---RETRYABLE---| +/* + - Errors: + Permanent: + - TaskAlreadyInitializedError (Permanent): Task is already intialized in the BLS Aggregation service (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L27). + - Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +*/ func (agg *Aggregator) InitializeNewTask(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { initilizeNewTask_func := func() error { - return agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, timeToExpiry) + err := agg.blsAggregationService.InitializeNewTask(batchIndex, taskCreatedBlock, quorumNums, quorumThresholdPercentages, timeToExpiry) + if err != nil { + // Task is already initialized + if strings.Contains(err.Error(), "already initialized") { + err = retry.PermanentError{Inner: err} + } + } + return err } - return retry.Retry(initilizeNewTask_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + return retry.Retry(initilizeNewTask_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) } // Long-lived goroutine that periodically checks and removes old Tasks from stored Maps diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 5e716d332..092b098ed 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/rpc" + "strings" "time" "github.com/Layr-Labs/eigensdk-go/crypto/bls" @@ -113,15 +114,34 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // |---RETRYABLE---| +/* + - Errors: + Permanent: + - TaskNotFoundError: Task corresponding to signature is not found within BLS Aggregation Service. (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L33) + - SignatureVerificationError: Verification of the sigature failed within the BLS Aggregation Service failed. (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L42) + in which case we stop retrying and pass error to higher context. + - Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +*/ func (agg *Aggregator) ProcessNewSignature(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { processNewSignature_func := func() error { - return agg.blsAggregationService.ProcessNewSignature( + err := agg.blsAggregationService.ProcessNewSignature( ctx, taskIndex, taskResponse, blsSignature, operatorId, ) + if err != nil { + // Task is not initialized + if strings.Contains(err.Error(), "not initialized or already completed") { + err = retry.PermanentError{Inner: err} + } + // Signature failed to process + if strings.Contains(err.Error(), "Failed to verify signature") { + err = retry.PermanentError{Inner: err} + } + } + return err } - return retry.Retry(processNewSignature_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + return retry.Retry(processNewSignature_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) } func (agg *Aggregator) GetTaskIndex(batchIdentifierHash [32]byte) (uint32, error) { diff --git a/core/chainio/retryable.go b/core/chainio/retryable.go index 059bcf109..09c53fb4a 100644 --- a/core/chainio/retryable.go +++ b/core/chainio/retryable.go @@ -3,7 +3,6 @@ package chainio import ( "context" "math/big" - "strings" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -16,26 +15,28 @@ import ( // |---AVS_WRITER---| +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +- NOTE: Contract call reverts are not considered `PermanentError`'s as block reorg's may lead to contract call revert in which case the aggregator should retry. +*/ func (w *AvsWriter) RespondToTaskV2Retryable(opts *bind.TransactOpts, batchMerkleRoot [32]byte, senderAddress common.Address, nonSignerStakesAndSignature servicemanager.IBLSSignatureCheckerNonSignerStakesAndSignature) (*types.Transaction, error) { - var ( - tx *types.Transaction - err error - ) respondToTaskV2_func := func() (*types.Transaction, error) { - tx, err = w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + // Try with main connection + tx, err := w.AvsContractBindings.ServiceManager.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) if err != nil { + // If error try with fallback tx, err = w.AvsContractBindings.ServiceManagerFallback.RespondToTaskV2(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) - if err != nil { - if strings.Contains(err.Error(), "execution reverted:") { - err = retry.PermanentError{Inner: err} - } - } } return tx, err } - return retry.RetryWithData(respondToTaskV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + return retry.RetryWithData(respondToTaskV2_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +*/ func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool @@ -47,30 +48,44 @@ func (w *AvsWriter) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (s Responded bool RespondToTaskFeeLimit *big.Int }, error) { + // Try with main connection state, err := w.AvsContractBindings.ServiceManager.BatchesState(opts, arg0) if err != nil { + // If error try with fallback connection state, err = w.AvsContractBindings.ServiceManagerFallback.BatchesState(opts, arg0) } return state, err } - return retry.RetryWithData(batchesState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + return retry.RetryWithData(batchesState_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +*/ func (w *AvsWriter) BatcherBalancesRetryable(opts *bind.CallOpts, senderAddress common.Address) (*big.Int, error) { batcherBalances_func := func() (*big.Int, error) { + // Try with main connection batcherBalance, err := w.AvsContractBindings.ServiceManager.BatchersBalances(opts, senderAddress) if err != nil { + // If error try with fallback connection batcherBalance, err = w.AvsContractBindings.ServiceManagerFallback.BatchersBalances(opts, senderAddress) } return batcherBalance, err } - return retry.RetryWithData(batcherBalances_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + return retry.RetryWithData(batcherBalances_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 1 sec, 2 sec, 4 sec. +*/ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress common.Address, blockNumber *big.Int) (*big.Int, error) { balanceAt_func := func() (*big.Int, error) { + // Try with main connection aggregatorBalance, err := w.Client.BalanceAt(ctx, aggregatorAddress, blockNumber) if err != nil { + // If error try with fallback connection aggregatorBalance, err = w.ClientFallback.BalanceAt(ctx, aggregatorAddress, blockNumber) } return aggregatorBalance, err @@ -80,10 +95,16 @@ func (w *AvsWriter) BalanceAtRetryable(ctx context.Context, aggregatorAddress co // |---AVS_SUBSCRIBER---| +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 1 sec, 2 sec, 4 sec. +*/ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error) { latestBlock_func := func() (uint64, error) { + // Try with main connection latestBlock, err := s.AvsContractBindings.ethClient.BlockNumber(ctx) if err != nil { + // If error try with fallback connection latestBlock, err = s.AvsContractBindings.ethClientFallback.BlockNumber(ctx) } return latestBlock, err @@ -91,6 +112,10 @@ func (s *AvsSubscriber) BlockNumberRetryable(ctx context.Context) (uint64, error return retry.RetryWithData(latestBlock_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 1 sec, 2 sec, 4 sec. +*/ func (s *AvsSubscriber) FilterBatchV2Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV2Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV2(opts, batchMerkleRoot) @@ -98,6 +123,10 @@ func (s *AvsSubscriber) FilterBatchV2Retryable(opts *bind.FilterOpts, batchMerkl return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 1 sec, 2 sec, 4 sec. +*/ func (s *AvsSubscriber) FilterBatchV3Retryable(opts *bind.FilterOpts, batchMerkleRoot [][32]byte) (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { filterNewBatchV2_func := func() (*servicemanager.ContractAlignedLayerServiceManagerNewBatchV3Iterator, error) { return s.AvsContractBindings.ServiceManager.FilterNewBatchV3(opts, batchMerkleRoot) @@ -105,6 +134,10 @@ func (s *AvsSubscriber) FilterBatchV3Retryable(opts *bind.FilterOpts, batchMerkl return retry.RetryWithData(filterNewBatchV2_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +*/ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte) (struct { TaskCreatedBlock uint32 Responded bool @@ -118,13 +151,19 @@ func (s *AvsSubscriber) BatchesStateRetryable(opts *bind.CallOpts, arg0 [32]byte return s.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.BatchesState(opts, arg0) } - return retry.RetryWithData(batchState_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + return retry.RetryWithData(batchState_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 1 sec, 2 sec, 4 sec. +*/ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- *types.Header) (ethereum.Subscription, error) { subscribeNewHead_func := func() (ethereum.Subscription, error) { + // Try with main connection sub, err := s.AvsContractBindings.ethClient.SubscribeNewHead(ctx, c) if err != nil { + // If error try with fallback connection sub, err = s.AvsContractBindings.ethClientFallback.SubscribeNewHead(ctx, c) } return sub, err @@ -132,6 +171,10 @@ func (s *AvsSubscriber) SubscribeNewHeadRetryable(ctx context.Context, c chan<- return retry.RetryWithData(subscribeNewHead_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 1 sec, 2 sec, 4 sec. +*/ func SubscribeToNewTasksV2Retryable( opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, @@ -144,6 +187,10 @@ func SubscribeToNewTasksV2Retryable( return retry.RetryWithData(subscribe_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) } +/* +- All errors are considered Transient Errors +- Retry times (3 retries): 1 sec, 2 sec, 4 sec. +*/ func SubscribeToNewTasksV3Retryable( opts *bind.WatchOpts, serviceManager *servicemanager.ContractAlignedLayerServiceManager, diff --git a/core/retry.go b/core/retry.go index c7cbf92ac..7ce08e930 100644 --- a/core/retry.go +++ b/core/retry.go @@ -7,11 +7,6 @@ import ( "github.com/cenkalti/backoff/v4" ) -/* -This retry library was inspired by and uses Cenk Alti (https://github.com/cenkalti) backoff library (https://github.com/cenkalti/backoff). -We would like to thank him for his great work. -*/ - /* Note we use a custom Permanent error type for asserting Permanent Erros within the retry library. We do not implement an explicit Transient error type and operate under the assumption that all errors that are not Permanent are Transient. @@ -30,13 +25,72 @@ func (e PermanentError) Is(err error) bool { } const ( - MinDelay = 1 * time.Second - MaxInterval = 60 * time.Second - MaxElapsedTime = 0 * time.Second - RetryFactor float64 = 2 - NumRetries uint64 = 3 + MinDelay = 1 * time.Second // Initial delay for retry interval. + MaxInterval = 60 * time.Second // Maximum interval an individual retry may have. + MaxElapsedTime = 0 * time.Second // Maximum time all retries may take. `0` corresponds to no limit on the time of the retries. + RetryFactor float64 = 2 // Multiplier factor computed exponential retry interval is scaled by. + NumRetries uint64 = 3 // Total number of retries attempted. + MinDelayChain = 12 * time.Second // Initial delay for retry interval for contract calls. Corresponds to 1 ethereum block. + MaxIntervalChain = 2 * time.Minute // Maximum interval for an individual retry. ) +/* +Retry and RetryWithData are custom retry functions used in Aligned's aggregator and operator to facilitate consistent retry logic across the system. +They are interfaces for around Cenk Alti (https://github.com/cenkalti) backoff library (https://github.com/cenkalti/backoff). We would like to thank him for his great work. + +The `Retry` and `RetryWithData` retry a supplied function at maximum `NumRetries` number of times. Upon execution, if the called function returns an error the retry library either re-executes the function (Transient Error) or exits and returns the error to the calling context (Permanent error) . +If the call is successful and no error is returned the library returns the result. `Permanent` errors are explicitly typed while `Transient` errors are implied by go's builtin error type. +For completeness: + +Transient: The error is recoverable and the function is retried after failing. `Transient` errors are do not have a defined error type and are implicitly defined by go's builtin `error` type. + +Permanenet: The error is not recoverable and the function is not retried exit to a larger context to handle the error. Permanent errors are explicitly typed and defined by wrapping the err within with `PermanentError`. + +Usage of `RetryWithData` is shown in the following example: +``` + sendUserMsg_func := func() (*types.Transaction, error) { + res, err := sendUserMessage(opts, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature) + if err != nil { + // Detect Permanent error by checking contents of returned error message. + if strings.Contains(err.Error(), "client error: User not registered:") { + err = retry.PermanentError{Inner: err} + } + } + return res, err + } + err := retry.Retry(sendUserMsg_func, retry.MinDelay, retry.RetryFactor, retry.NumRetries, retry.MaxInterval, retry.MaxElapsedTime) + if err != nil { + fmt.Printf("error: %v\n", err) + } +``` + +# Retry Intervals: +The backoff period for each retry attempt increases using a randomization function that grows exponentially. + +retryinterval = + currentRetryinterval * (random value in range [1 - randomizationfactor, 1 + randomizationfactor]) * retryFactor + +This library defaults to the use of the following parameters: + +randomizationFactor = 0.5 // Randomization factor that maps the interval increase within a range around the computed retry interval. +initialRetryInterval = 1 sec // Initial value used in the retry interval +Multiplier = 2 // Multiplier used to scale the values. + +# Default intervals for Retries (sec) +request retry_interval (1 sec) randomized_interval (0.5) randomized_interval_scaled (2) + 1 1 [0.5, 1.5] [1, 3] + 2 2 [1, 3] [2, 6] + 3 4 [2, 6] [4, 12] + +# Default intervals for Contract Calls (sec) +request retry_interval (12 sec) randomized_interval (0.5) randomized_interval_scaled (2) + 1 12 [6, 18] [12, 36] + 2 24 [12, 36] [24, 72] + 3 48 [24, 72]` [48, 144]` + +Reference: https://github.com/cenkalti/backoff/blob/v4/exponential.go#L9 +*/ + // Same as Retry only that the functionToRetry can return a value upon correct execution func RetryWithData[T any](functionToRetry func() (T, error), minDelay time.Duration, factor float64, maxTries uint64, maxInterval time.Duration, maxElapsedTime time.Duration) (T, error) { f := func() (T, error) { From 03398a19bb1d1efbba54b50cb1e8ecdd38a3b05d Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:04:41 -0300 Subject: [PATCH 123/135] fix: CI test go retries --- .github/workflows/test-go-retries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-go-retries.yml b/.github/workflows/test-go-retries.yml index 79f1cfe97..175ea1067 100644 --- a/.github/workflows/test-go-retries.yml +++ b/.github/workflows/test-go-retries.yml @@ -31,6 +31,6 @@ jobs: with: toolchain: stable - name: foundry-toolchain - uses: foundry-rs/foundry-toolchain@v1.2.0 + uses: foundry-rs/foundry-toolchain@v1.2.0 - name: Test go Retry Functions run: make test_go_retries \ No newline at end of file From 5c02e1d109ccf5bc9be04d7d15e81cda6418e328 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:05:24 -0300 Subject: [PATCH 124/135] fix: run make test on build and test go ci --- .github/workflows/build-and-test-go.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-test-go.yml b/.github/workflows/build-and-test-go.yml index 7abf54ec4..5aff66dff 100644 --- a/.github/workflows/build-and-test-go.yml +++ b/.github/workflows/build-and-test-go.yml @@ -38,3 +38,5 @@ jobs: run: go build operator/cmd/main.go - name: Build aggregator run: go build aggregator/cmd/main.go + - name: Test Go + run: make test From c2acff660865e6db1beef3562fc25709c0caf792 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 11 Nov 2024 15:54:37 -0300 Subject: [PATCH 125/135] remove permanent erros for ProcessNewSignature + comment --- aggregator/pkg/server.go | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 092b098ed..77d025b37 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "net/rpc" - "strings" "time" "github.com/Layr-Labs/eigensdk-go/crypto/bls" @@ -115,30 +114,16 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // |---RETRYABLE---| /* - - Errors: - Permanent: - - TaskNotFoundError: Task corresponding to signature is not found within BLS Aggregation Service. (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L33) - - SignatureVerificationError: Verification of the sigature failed within the BLS Aggregation Service failed. (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L42) - in which case we stop retrying and pass error to higher context. - - Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +- All errors are considered Transient Errors +- Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) +- NOTE: TaskNotFound and SignatureVerificationFailed errors from the BLS Aggregation service are Transient errors as block reorg's may lead to these errors being thrown. */ func (agg *Aggregator) ProcessNewSignature(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { processNewSignature_func := func() error { - err := agg.blsAggregationService.ProcessNewSignature( + return agg.blsAggregationService.ProcessNewSignature( ctx, taskIndex, taskResponse, blsSignature, operatorId, ) - if err != nil { - // Task is not initialized - if strings.Contains(err.Error(), "not initialized or already completed") { - err = retry.PermanentError{Inner: err} - } - // Signature failed to process - if strings.Contains(err.Error(), "Failed to verify signature") { - err = retry.PermanentError{Inner: err} - } - } - return err } return retry.Retry(processNewSignature_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) From bb89e0512387c75dfafbb88fd828393f33b7f624 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 11 Nov 2024 16:27:03 -0300 Subject: [PATCH 126/135] adjust timeout for retry tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cc8e40016..d25a5fe46 100644 --- a/Makefile +++ b/Makefile @@ -145,7 +145,7 @@ aggregator_send_dummy_responses: test_go_retries: @cd core/ && \ - go test -v + go test -v -timeout 15m __OPERATOR__: From 0c47ec220e1c135fd754e4b67436ff8269593934 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 11 Nov 2024 16:34:47 -0300 Subject: [PATCH 127/135] add back in signature process failure for ProcessNewSignature --- aggregator/pkg/aggregator.go | 2 ++ aggregator/pkg/server.go | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index eb4a108f6..2866066f9 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -388,6 +388,8 @@ func (agg *Aggregator) AddNewTask(batchMerkleRoot [32]byte, senderAddress [20]by - Errors: Permanent: - TaskAlreadyInitializedError (Permanent): Task is already intialized in the BLS Aggregation service (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L27). + Transient: + - All others. - Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) */ func (agg *Aggregator) InitializeNewTask(batchIndex uint32, taskCreatedBlock uint32, quorumNums eigentypes.QuorumNums, quorumThresholdPercentages eigentypes.QuorumThresholdPercentages, timeToExpiry time.Duration) error { diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index 77d025b37..ee770246b 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/rpc" + "strings" "time" "github.com/Layr-Labs/eigensdk-go/crypto/bls" @@ -114,16 +115,26 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { // |---RETRYABLE---| /* -- All errors are considered Transient Errors -- Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) -- NOTE: TaskNotFound and SignatureVerificationFailed errors from the BLS Aggregation service are Transient errors as block reorg's may lead to these errors being thrown. + - Errors: + Permanent: + - SignatureVerificationError: Verification of the sigature failed within the BLS Aggregation Service failed. (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L42). + Transient: + - All others. + - Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) + - NOTE: TaskNotFound errors from the BLS Aggregation service are Transient errors as block reorg's may lead to these errors being thrown. */ func (agg *Aggregator) ProcessNewSignature(ctx context.Context, taskIndex uint32, taskResponse interface{}, blsSignature *bls.Signature, operatorId eigentypes.Bytes32) error { processNewSignature_func := func() error { - return agg.blsAggregationService.ProcessNewSignature( + err := agg.blsAggregationService.ProcessNewSignature( ctx, taskIndex, taskResponse, blsSignature, operatorId, ) + if err != nil { + if strings.Contains(err.Error(), "Failed to verify signature") { + err = retry.PermanentError{Inner: err} + } + } + return err } return retry.Retry(processNewSignature_func, retry.MinDelayChain, retry.RetryFactor, retry.NumRetries, retry.MaxIntervalChain, retry.MaxElapsedTime) From 45a0c4d06dec2e3b790a11963e9be4a8b813d034 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 11 Nov 2024 16:51:51 -0300 Subject: [PATCH 128/135] add anvil to make test --- .github/workflows/build-and-test-go.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-test-go.yml b/.github/workflows/build-and-test-go.yml index 5aff66dff..63d9d2b98 100644 --- a/.github/workflows/build-and-test-go.yml +++ b/.github/workflows/build-and-test-go.yml @@ -24,6 +24,8 @@ jobs: with: go-version: "1.23" cache: false + - name: foundry-toolchain + uses: foundry-rs/foundry-toolchain@v1.2.0 - name: Build SP1 bindings run: make build_sp1_linux - name: Build Old SP1 bindings From 823ccf1d6172c1db3fddf6698ae1bc8a0e8399ad Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 11 Nov 2024 16:57:46 -0300 Subject: [PATCH 129/135] add documentation nits --- aggregator/pkg/server.go | 2 +- core/retry.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aggregator/pkg/server.go b/aggregator/pkg/server.go index ee770246b..0c3ee5c8b 100644 --- a/aggregator/pkg/server.go +++ b/aggregator/pkg/server.go @@ -117,7 +117,7 @@ func (agg *Aggregator) ServerRunning(_ *struct{}, reply *int64) error { /* - Errors: Permanent: - - SignatureVerificationError: Verification of the sigature failed within the BLS Aggregation Service failed. (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L42). + - SignatureVerificationError: Verification of the sigature within the BLS Aggregation Service failed. (https://github.com/Layr-Labs/eigensdk-go/blob/dev/services/bls_aggregation/blsagg.go#L42). Transient: - All others. - Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks) diff --git a/core/retry.go b/core/retry.go index 7ce08e930..551be59c7 100644 --- a/core/retry.go +++ b/core/retry.go @@ -42,9 +42,9 @@ The `Retry` and `RetryWithData` retry a supplied function at maximum `NumRetries If the call is successful and no error is returned the library returns the result. `Permanent` errors are explicitly typed while `Transient` errors are implied by go's builtin error type. For completeness: -Transient: The error is recoverable and the function is retried after failing. `Transient` errors are do not have a defined error type and are implicitly defined by go's builtin `error` type. +Transient: The error is recoverable and the function is retried after failing. `Transient` errors do not have a defined error type and are implicitly defined by go's builtin `error` type. -Permanenet: The error is not recoverable and the function is not retried exit to a larger context to handle the error. Permanent errors are explicitly typed and defined by wrapping the err within with `PermanentError`. +Permanent: The error is not recoverable by retrying and the error to the calling context. Permanent errors are explicitly typed and defined by wrapping the err within with `PermanentError` type. Usage of `RetryWithData` is shown in the following example: ``` From 2a17a150fb2ce79897ec120c4dce2beba4409071 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 11 Nov 2024 17:20:34 -0300 Subject: [PATCH 130/135] adjust timeout in go test for make test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d25a5fe46..9a8ed7720 100644 --- a/Makefile +++ b/Makefile @@ -195,7 +195,7 @@ bindings: cd contracts && ./generate-go-bindings.sh test: - go test ./... + go test ./... -timeout 15m get_delegation_manager_address: From 7847bc2e6d1d5791f08ff94dbbba4887245d6f27 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 12 Nov 2024 10:50:08 -0300 Subject: [PATCH 131/135] style: fmt yaml --- .github/workflows/build-and-test-go.yml | 16 ++++++++-------- config-files/config-aggregator.yaml | 24 ++++++++++++------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-and-test-go.yml b/.github/workflows/build-and-test-go.yml index f6ac29a28..df306e755 100644 --- a/.github/workflows/build-and-test-go.yml +++ b/.github/workflows/build-and-test-go.yml @@ -4,14 +4,14 @@ on: push: branches: [main] pull_request: - branches: ['*'] + branches: ["*"] paths: - - 'operator/**' - - 'aggregator/**' - - 'common/**' - - 'core/**' - - 'metrics/**' - - '.github/workflows/build-go.yml' + - "operator/**" + - "aggregator/**" + - "common/**" + - "core/**" + - "metrics/**" + - ".github/workflows/build-go.yml" env: FFI_FOR_RELEASE: false jobs: @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version: "1.23" cache: false - name: foundry-toolchain uses: foundry-rs/foundry-toolchain@v1.2.0 diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 64421ba33..7ce0c2356 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -1,23 +1,23 @@ # Common variables for all the services # 'production' only prints info and above. 'development' also prints debug -environment: 'production' -aligned_layer_deployment_config_file_path: './contracts/script/output/devnet/alignedlayer_deployment_output.json' -eigen_layer_deployment_config_file_path: './contracts/script/output/devnet/eigenlayer_deployment_output.json' -eth_rpc_url: 'http://localhost:8545' -eth_rpc_url_fallback: 'http://localhost:8545' -eth_ws_url: 'ws://localhost:8545' -eth_ws_url_fallback: 'ws://localhost:8545' -eigen_metrics_ip_port_address: 'localhost:9090' +environment: "production" +aligned_layer_deployment_config_file_path: "./contracts/script/output/devnet/alignedlayer_deployment_output.json" +eigen_layer_deployment_config_file_path: "./contracts/script/output/devnet/eigenlayer_deployment_output.json" +eth_rpc_url: "http://localhost:8545" +eth_rpc_url_fallback: "http://localhost:8545" +eth_ws_url: "ws://localhost:8545" +eth_ws_url_fallback: "ws://localhost:8545" +eigen_metrics_ip_port_address: "localhost:9090" ## ECDSA Configurations ecdsa: - private_key_store_path: 'config-files/anvil.aggregator.ecdsa.key.json' - private_key_store_password: '' + private_key_store_path: "config-files/anvil.aggregator.ecdsa.key.json" + private_key_store_password: "" ## BLS Configurations bls: - private_key_store_path: 'config-files/anvil.aggregator.bls.key.json' - private_key_store_password: '' + private_key_store_path: "config-files/anvil.aggregator.bls.key.json" + private_key_store_password: "" # ## Batcher configurations # batcher: # block_interval: 3 From c5156810abc03633902b5078437f0abe7451f17b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 12 Nov 2024 11:43:03 -0300 Subject: [PATCH 132/135] fix: dashboards --- .../aligned/aggregator_batcher.json | 1137 +++++++++-------- 1 file changed, 571 insertions(+), 566 deletions(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index 30849bcda..a7fc54543 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -1116,7 +1116,7 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -1124,611 +1124,616 @@ "y": 33 }, "id": 10, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red" - }, - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "dark-red", - "value": 2 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 10, - "x": 0, - "y": 58 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "panels": [], + "title": "Aggregator", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "floor(increase(aligned_aggregator_received_tasks{job=\"aligned-aggregator\"}[$__range]))", - "fullMetaSearch": false, - "hide": true, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + { + "color": "green", + "value": 0 }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[$__range]))", - "fullMetaSearch": false, - "hide": true, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "name": "Expression", - "type": "__expr__", - "uid": "__expr__" + { + "color": "yellow", + "value": 1 }, - "expression": "$A - $B", - "hide": false, - "reducer": "last", - "refId": "C", - "type": "math" - } - ], - "title": "Tasks Not Verified", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "lower", - "options": { - "value": 0 - } - }, - "fieldName": "C {bot=\"aggregator\", instance=\"host.docker.internal:9091\", job=\"aligned-aggregator\"}" - } - ], - "match": "any", - "type": "exclude" + { + "color": "dark-red", + "value": 2 } - } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 34 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "", + "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.1.10", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 10, - "x": 10, - "y": 58 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[10y]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Verified Tasks", - "type": "timeseries" + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "floor(increase(aligned_aggregator_received_tasks{job=\"aligned-aggregator\"}[$__range]))", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A", + "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 0, - "y": 65 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "floor(increase(aligned_aggregator_received_tasks{bot=\"aggregator\"}[10y]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Total Tasks Received", - "type": "stat" + "disableTextWrap": false, + "editorMode": "builder", + "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[$__range]))", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false }, { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 + "expression": "$A - $B", + "hide": false, + "reducer": "last", + "refId": "C", + "type": "math" + } + ], + "title": "Tasks Not Verified", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "lower", + "options": { + "value": 0 } - ] + }, + "fieldName": "C {bot=\"aggregator\", instance=\"host.docker.internal:9091\", job=\"aligned-aggregator\"}" } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 5, - "y": 65 + ], + "match": "any", + "type": "exclude" + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "floor(increase(aligned_aggregator_received_tasks{bot=\"aggregator\"}[$__range]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Tasks Received", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Number of times gas price was bumped while sending an aggregated response.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 10, - "x": 10, - "y": 35 - }, - "id": 21, - "interval": "36", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "aligned_respond_to_task_gas_price_bumped{bot=\"aggregator\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Bumped gas price for aggregated responses", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] + { + "color": "red", + "value": 80 } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 0, - "y": 72 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[10y]))", - "format": "table", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Total Tasks Verified", - "type": "stat" + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 10, + "y": 34 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 5, - "y": 72 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[$__range]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } + "disableTextWrap": false, + "editorMode": "builder", + "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[10y]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Verified Tasks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 41 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Tasks Verified", - "type": "stat" + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "floor(increase(aligned_aggregator_received_tasks{bot=\"aggregator\"}[10y]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false } ], - "title": "Aggregator", - "type": "row" + "title": "Total Tasks Received", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 41 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "floor(increase(aligned_aggregator_received_tasks{bot=\"aggregator\"}[$__range]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tasks Received", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of times gas price was bumped while sending an aggregated response.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 10, + "y": 41 + }, + "id": 25, + "interval": "36", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aligned_respond_to_task_gas_price_bumped{bot=\"aggregator\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Bumped gas price for aggregated responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 48 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[10y]))", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Tasks Verified", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 48 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.1.10", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "floor(increase(aligned_aggregated_responses{bot=\"aggregator\"}[$__range]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tasks Verified", + "type": "stat" } ], "refresh": "5s", @@ -1746,6 +1751,6 @@ "timezone": "browser", "title": "Aggregator Data", "uid": "aggregator", - "version": 3, + "version": 8, "weekStart": "" } From 475125db9e982b3e220a055c0ffe29b3d76dc8db Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 12 Nov 2024 16:15:33 -0300 Subject: [PATCH 133/135] refactor: wait timeouts --- core/chainio/avs_writer.go | 2 +- core/utils/eth_client_utils.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 56c528b21..67d65dcba 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -142,7 +142,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, fmt.Errorf("transaction failed") } - return retry.RetryWithData(respondToTaskV2Func, 1000, 2, 0, 60000, retry.MaxElapsedTime) + return retry.RetryWithData(respondToTaskV2Func, retry.MinDelay, retry.RetryFactor, 0, retry.MaxInterval, 0) } func (w *AvsWriter) checkRespondToTaskFeeLimit(tx *types.Transaction, txOpts bind.TransactOpts, batchIdentifierHash [32]byte, senderAddress [20]byte) error { diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 727944787..0b36b5f2e 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -15,11 +15,14 @@ import ( // WaitForTransactionReceiptRetryable repeatedly attempts to fetch the transaction receipt for a given transaction hash. // If the receipt is not found, the function will retry with exponential backoff until the specified `waitTimeout` duration is reached. // If the receipt is still unavailable after `waitTimeout`, it will return an error. +// +// Note: The `time.Second * 2` is set as the max interval in the retry mechanism because we can't reliably measure the specific time the tx will be included in a block. +// Setting a higher value will imply doing less retries across the waitTimeout and so we might lose the receipt func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash, waitTimeout time.Duration) (*types.Receipt, error) { receipt_func := func() (*types.Receipt, error) { return client.TransactionReceipt(ctx, txHash) } - return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, 0, retry.MaxInterval, waitTimeout) + return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, 0, time.Second*2, waitTimeout) } func BytesToQuorumNumbers(quorumNumbersBytes []byte) eigentypes.QuorumNums { From 0c46f52eb16c760a42594cf777ead4b9a60a8428 Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Tue, 12 Nov 2024 16:49:50 -0300 Subject: [PATCH 134/135] style: fmt --- config-files/config-aggregator.yaml | 2 +- core/config/aggregator.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config-files/config-aggregator.yaml b/config-files/config-aggregator.yaml index 7ce0c2356..ce1fba5c5 100644 --- a/config-files/config-aggregator.yaml +++ b/config-files/config-aggregator.yaml @@ -39,7 +39,7 @@ aggregator: garbage_collector_tasks_interval: 10 #The interval of queried blocks to get an old batch. Suggested value for prod: '900' (3 hours) bls_service_task_timeout: 168h # The timeout of bls aggregation service tasks. Suggested value for prod '168h' (7 days) gas_base_bump_percentage: 10 # How much to bump gas price when responding to task. Suggested value 10% - gas_bump_incremental_percentage: 5 # An extra percentage to bump every retry i*5 when responding to task. Suggested value 2% + gas_bump_incremental_percentage: 2 # An extra percentage to bump every retry i*2 when responding to task. Suggested value 2% time_to_wait_before_bump: 36s # The time to wait for the receipt when responding to task. Suggested value 36 seconds (3 blocks) ## Operator Configurations diff --git a/core/config/aggregator.go b/core/config/aggregator.go index 29340c470..a55da8193 100644 --- a/core/config/aggregator.go +++ b/core/config/aggregator.go @@ -24,7 +24,7 @@ type AggregatorConfig struct { GarbageCollectorPeriod time.Duration GarbageCollectorTasksAge uint64 GarbageCollectorTasksInterval uint64 - BlsServiceTaskTimeout time.Duration + BlsServiceTaskTimeout time.Duration GasBaseBumpPercentage uint GasBumpIncrementalPercentage uint TimeToWaitBeforeBump time.Duration @@ -42,7 +42,7 @@ type AggregatorConfigFromYaml struct { GarbageCollectorPeriod time.Duration `yaml:"garbage_collector_period"` GarbageCollectorTasksAge uint64 `yaml:"garbage_collector_tasks_age"` GarbageCollectorTasksInterval uint64 `yaml:"garbage_collector_tasks_interval"` - BlsServiceTaskTimeout time.Duration `yaml:"bls_service_task_timeout"` + BlsServiceTaskTimeout time.Duration `yaml:"bls_service_task_timeout"` GasBaseBumpPercentage uint `yaml:"gas_base_bump_percentage"` GasBumpIncrementalPercentage uint `yaml:"gas_bump_incremental_percentage"` TimeToWaitBeforeBump time.Duration `yaml:"time_to_wait_before_bump"` @@ -90,7 +90,7 @@ func NewAggregatorConfig(configFilePath string) *AggregatorConfig { GarbageCollectorPeriod time.Duration GarbageCollectorTasksAge uint64 GarbageCollectorTasksInterval uint64 - BlsServiceTaskTimeout time.Duration + BlsServiceTaskTimeout time.Duration GasBaseBumpPercentage uint GasBumpIncrementalPercentage uint TimeToWaitBeforeBump time.Duration From 5b1219a79927bf1c66394d9eed4a52cff2259ddf Mon Sep 17 00:00:00 2001 From: Marcos Nicolau Date: Wed, 13 Nov 2024 12:37:43 -0300 Subject: [PATCH 135/135] refactor: address review comments --- aggregator/pkg/aggregator.go | 12 +++++++++++- core/chainio/avs_writer.go | 4 ++-- core/retry_test.go | 6 +++--- core/utils/eth_client_utils.go | 24 +++++++++++++++++++----- core/utils/eth_client_utils_test.go | 4 ++-- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/aggregator/pkg/aggregator.go b/aggregator/pkg/aggregator.go index 02f6751ca..0d8cbd9cb 100644 --- a/aggregator/pkg/aggregator.go +++ b/aggregator/pkg/aggregator.go @@ -301,11 +301,21 @@ func (agg *Aggregator) sendAggregatedResponse(batchIdentifierHash [32]byte, batc "senderAddress", hex.EncodeToString(senderAddress[:]), "batchIdentifierHash", hex.EncodeToString(batchIdentifierHash[:])) + // This function is a callback that is called when the gas price is bumped on the avsWriter.SendAggregatedResponse onGasPriceBumped := func(bumpedGasPrice *big.Int) { agg.metrics.IncBumpedGasPriceForAggregatedResponse() agg.telemetry.BumpedTaskGasPrice(batchMerkleRoot, bumpedGasPrice.String()) } - receipt, err := agg.avsWriter.SendAggregatedResponse(batchIdentifierHash, batchMerkleRoot, senderAddress, nonSignerStakesAndSignature, agg.AggregatorConfig.Aggregator.GasBaseBumpPercentage, agg.AggregatorConfig.Aggregator.GasBumpIncrementalPercentage, agg.AggregatorConfig.Aggregator.TimeToWaitBeforeBump, onGasPriceBumped) + receipt, err := agg.avsWriter.SendAggregatedResponse( + batchIdentifierHash, + batchMerkleRoot, + senderAddress, + nonSignerStakesAndSignature, + agg.AggregatorConfig.Aggregator.GasBaseBumpPercentage, + agg.AggregatorConfig.Aggregator.GasBumpIncrementalPercentage, + agg.AggregatorConfig.Aggregator.TimeToWaitBeforeBump, + onGasPriceBumped, + ) if err != nil { agg.walletMutex.Unlock() agg.logger.Infof("- Unlocked Wallet Resources: Error sending aggregated response for batch %s. Error: %s", hex.EncodeToString(batchIdentifierHash[:]), err) diff --git a/core/chainio/avs_writer.go b/core/chainio/avs_writer.go index 67d65dcba..012e069bb 100644 --- a/core/chainio/avs_writer.go +++ b/core/chainio/avs_writer.go @@ -96,7 +96,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe i := 0 respondToTaskV2Func := func() (*types.Receipt, error) { - gasPrice, err := utils.GetGasPriceRetryable(w.Client, context.Background()) + gasPrice, err := utils.GetGasPriceRetryable(w.Client, w.ClientFallback) if err != nil { return nil, err } @@ -126,7 +126,7 @@ func (w *AvsWriter) SendAggregatedResponse(batchIdentifierHash [32]byte, batchMe return nil, err } - receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, context.Background(), tx.Hash(), timeToWaitBeforeBump) + receipt, err := utils.WaitForTransactionReceiptRetryable(w.Client, w.ClientFallback, tx.Hash(), timeToWaitBeforeBump) if receipt != nil { return receipt, nil } diff --git a/core/retry_test.go b/core/retry_test.go index 7d90c665f..c8e7fbfb2 100644 --- a/core/retry_test.go +++ b/core/retry_test.go @@ -153,7 +153,7 @@ func TestWaitForTransactionReceipt(t *testing.T) { } // Assert Call succeeds when Anvil running - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash, time.Second*45) + _, err = utils.WaitForTransactionReceiptRetryable(*client, *client, hash, time.Second*45) assert.NotNil(t, err, "Error Waiting for Transaction with Anvil Running: %s\n", err) if !strings.Contains(err.Error(), "not found") { t.Errorf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) @@ -165,7 +165,7 @@ func TestWaitForTransactionReceipt(t *testing.T) { return } - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash, time.Second*45) + _, err = utils.WaitForTransactionReceiptRetryable(*client, *client, hash, time.Second*45) assert.NotNil(t, err) if _, ok := err.(retry.PermanentError); ok { t.Errorf("WaitForTransactionReceipt Emitted non Transient error: %s\n", err) @@ -181,7 +181,7 @@ func TestWaitForTransactionReceipt(t *testing.T) { t.Errorf("Error setting up Anvil: %s\n", err) } - _, err = utils.WaitForTransactionReceiptRetryable(*client, context.Background(), hash, time.Second*45) + _, err = utils.WaitForTransactionReceiptRetryable(*client, *client, hash, time.Second*45) assert.NotNil(t, err) if !strings.Contains(err.Error(), "not found") { t.Errorf("WaitForTransactionReceipt Emitted incorrect error: %s\n", err) diff --git a/core/utils/eth_client_utils.go b/core/utils/eth_client_utils.go index 0b36b5f2e..6f71ff3e4 100644 --- a/core/utils/eth_client_utils.go +++ b/core/utils/eth_client_utils.go @@ -18,9 +18,17 @@ import ( // // Note: The `time.Second * 2` is set as the max interval in the retry mechanism because we can't reliably measure the specific time the tx will be included in a block. // Setting a higher value will imply doing less retries across the waitTimeout and so we might lose the receipt -func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, ctx context.Context, txHash gethcommon.Hash, waitTimeout time.Duration) (*types.Receipt, error) { +func WaitForTransactionReceiptRetryable(client eth.InstrumentedClient, fallbackClient eth.InstrumentedClient, txHash gethcommon.Hash, waitTimeout time.Duration) (*types.Receipt, error) { receipt_func := func() (*types.Receipt, error) { - return client.TransactionReceipt(ctx, txHash) + receipt, err := client.TransactionReceipt(context.Background(), txHash) + if err != nil { + receipt, err = client.TransactionReceipt(context.Background(), txHash) + if err != nil { + return nil, err + } + return receipt, nil + } + return receipt, nil } return retry.RetryWithData(receipt_func, retry.MinDelay, retry.RetryFactor, 0, time.Second*2, waitTimeout) } @@ -45,7 +53,7 @@ func BytesToQuorumThresholdPercentages(quorumThresholdPercentagesBytes []byte) e // the currentGasPrice, a base bump percentage, a retry percentage, and the retry count. // Formula: currentGasPrice + (currentGasPrice * (baseBumpPercentage + retryCount * incrementalRetryPercentage) / 100) func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercentage uint, retryAttemptPercentage uint, retryCount int) *big.Int { - // Incremental percentage increase for each retry attempt (i*5%) + // Incremental percentage increase for each retry attempt (i*retryAttemptPercentage) incrementalRetryPercentage := new(big.Int).Mul(big.NewInt(int64(retryAttemptPercentage)), big.NewInt(int64(retryCount))) // Total bump percentage: base bump + incremental retry percentage @@ -61,11 +69,17 @@ func CalculateGasPriceBumpBasedOnRetry(currentGasPrice *big.Int, baseBumpPercent return bumpedGasPrice } -func GetGasPriceRetryable(client eth.InstrumentedClient, ctx context.Context) (*big.Int, error) { +/* +GetGasPriceRetryable +Get the gas price from the client with retry logic. +- All errors are considered Transient Errors +- Retry times: 1 sec, 2 sec, 4 sec +*/ +func GetGasPriceRetryable(client eth.InstrumentedClient, fallbackClient eth.InstrumentedClient) (*big.Int, error) { respondToTaskV2_func := func() (*big.Int, error) { gasPrice, err := client.SuggestGasPrice(context.Background()) if err != nil { - gasPrice, err = client.SuggestGasPrice(context.Background()) + gasPrice, err = fallbackClient.SuggestGasPrice(context.Background()) if err != nil { return nil, err } diff --git a/core/utils/eth_client_utils_test.go b/core/utils/eth_client_utils_test.go index 11d57150f..277fa0ba0 100644 --- a/core/utils/eth_client_utils_test.go +++ b/core/utils/eth_client_utils_test.go @@ -8,8 +8,8 @@ import ( ) func TestCalculateGasPriceBumpBasedOnRetry(t *testing.T) { - incrementalRetryPercentage := uint(20) - baseBumpPercentage := uint(5) + baseBumpPercentage := uint(20) + incrementalRetryPercentage := uint(5) gasPrices := [5]*big.Int{ big.NewInt(3000000000),