From 971371621debd8444e0a4a476d2c92596caa56cb Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 10:08:24 -0300 Subject: [PATCH 01/30] fix: add request_timeout to explorer --- explorer/lib/explorer/periodically.ex | 54 +++++++++++++------------ explorer/lib/explorer_web/live/utils.ex | 34 +++++++++------- 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index 423b3738e..54834cf2a 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -79,32 +79,36 @@ defmodule Explorer.Periodically do {:ok, lock} -> "Processing batch: #{batch.merkle_root}" |> Logger.debug() - {batch_changeset, proofs} = - batch - |> Utils.extract_info_from_data_pointer() - |> Batches.generate_changesets() - - Batches.insert_or_update(batch_changeset, proofs) - |> case do - {:ok, _} -> - PubSub.broadcast(Explorer.PubSub, "update_views", %{ - eth_usd: - case EthConverter.get_eth_price_usd() do - {:ok, eth_usd_price} -> eth_usd_price - {:error, _error} -> :empty - end - }) - - {:error, error} -> - Logger.error("Some error in DB operation, not broadcasting update_views: #{inspect(error)}") - - # no changes in DB - nil -> - nil + with {:ok, batch_info} <- Utils.extract_info_from_data_pointer(batch), + {batch_changeset, proofs} <- Batches.generate_changesets(batch_info) do + Batches.insert_or_update(batch_changeset, proofs) + |> case do + {:ok, _} -> + PubSub.broadcast(Explorer.PubSub, "update_views", %{ + eth_usd: + case EthConverter.get_eth_price_usd() do + {:ok, eth_usd_price} -> eth_usd_price + {:error, _error} -> :empty + end + }) + + {:error, error} -> + Logger.error( + "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" + ) + + # no changes in DB + nil -> + nil + end + + "Done processing batch: #{batch.merkle_root}" |> Logger.debug() + Mutex.release(BatchMutex, lock) + else + {:error, {:http_error, reason}} -> + Logger.error("Error when procesing request body: #{inspect(reason)}") + # Maybe delete the batch end - - "Done processing batch: #{batch.merkle_root}" |> Logger.debug() - Mutex.release(BatchMutex, lock) end end diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index ae4c50382..5145f8571 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -168,19 +168,20 @@ defmodule Utils do end def calculate_proof_hashes({:ok, deserialized_batch}) do - deserialized_batch - |> Enum.map(fn s3_object -> - :crypto.hash(:sha3_256, s3_object["proof"]) - end) - end + proof_hashes = + deserialized_batch + |> Enum.map(fn s3_object -> + :crypto.hash(:sha3_256, s3_object["proof"]) + end) - def calculate_proof_hashes({:error, reason}) do - Logger.error("Error calculating proof hashes: #{inspect(reason)}") - [] + {:ok, proof_hashes} end + def calculate_proof_hashes(error), do: error + def fetch_batch_data_pointer(batch_data_pointer) do - case Finch.build(:get, batch_data_pointer) |> Finch.request(Explorer.Finch) do + case Finch.build(:get, batch_data_pointer) + |> Finch.request(Explorer.Finch, request_timeout: 10_000) do {:ok, %Finch.Response{status: 200, body: body}} -> cond do is_json?(body) -> @@ -240,7 +241,7 @@ defmodule Utils do def extract_info_from_data_pointer(%BatchDB{} = batch) do Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") # only get from s3 if not already in DB - proof_hashes = + result = case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do nil -> Logger.debug("Fetching from S3") @@ -252,12 +253,17 @@ defmodule Utils do proof_hashes -> # already processed and stored the S3 data Logger.debug("Fetching from DB") - proof_hashes + {:ok, proof_hashes} end - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) + with {:ok, proof_hashes} <- result do + batch_info = + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) + + {:ok, batch_info} + end end def fetch_eigen_operator_metadata(url) do From ba51f1cbc29ec2f179606d5a4380ccb5c706213a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 10:13:48 -0300 Subject: [PATCH 02/30] refactor: replace the value with a constant --- explorer/lib/explorer_web/live/utils.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 5145f8571..a31033bb2 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -1,5 +1,7 @@ # Frontend Utils defmodule ExplorerWeb.Helpers do + @request_timeout 10_000 + def shorten_hash(hash, decimals \\ 6) do case String.length(hash) do n when n < decimals -> hash @@ -181,7 +183,7 @@ defmodule Utils do def fetch_batch_data_pointer(batch_data_pointer) do case Finch.build(:get, batch_data_pointer) - |> Finch.request(Explorer.Finch, request_timeout: 10_000) do + |> Finch.request(Explorer.Finch, request_timeout: @request_timeout) do {:ok, %Finch.Response{status: 200, body: body}} -> cond do is_json?(body) -> From 9e1069713350eea61764c4b9e6e4ca884cef260b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:19:19 -0300 Subject: [PATCH 03/30] refactor: handle error --- explorer/lib/explorer/periodically.ex | 63 ++++++++++++++------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index 54834cf2a..d1b54df38 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -16,8 +16,10 @@ defmodule Explorer.Periodically do one_second = 1000 seconds_in_an_hour = 60 * 60 - :timer.send_interval(one_second * 12, :batches) # every 12 seconds, once per block - :timer.send_interval(one_second * seconds_in_an_hour, :restakings) # every 1 hour + # every 12 seconds, once per block + :timer.send_interval(one_second * 12, :batches) + # every 1 hour + :timer.send_interval(one_second * seconds_in_an_hour, :restakings) end # Reads and process last blocks for operators and restaking changes @@ -45,6 +47,7 @@ defmodule Explorer.Periodically do run_every_n_iterations = 8 new_count = rem(count + 1, run_every_n_iterations) + if new_count == 0 do Task.start(&process_unverified_batches/0) end @@ -79,36 +82,34 @@ defmodule Explorer.Periodically do {:ok, lock} -> "Processing batch: #{batch.merkle_root}" |> Logger.debug() - with {:ok, batch_info} <- Utils.extract_info_from_data_pointer(batch), - {batch_changeset, proofs} <- Batches.generate_changesets(batch_info) do - Batches.insert_or_update(batch_changeset, proofs) - |> case do - {:ok, _} -> - PubSub.broadcast(Explorer.PubSub, "update_views", %{ - eth_usd: - case EthConverter.get_eth_price_usd() do - {:ok, eth_usd_price} -> eth_usd_price - {:error, _error} -> :empty - end - }) - - {:error, error} -> - Logger.error( - "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" - ) - - # no changes in DB - nil -> - nil - end - - "Done processing batch: #{batch.merkle_root}" |> Logger.debug() - Mutex.release(BatchMutex, lock) - else - {:error, {:http_error, reason}} -> - Logger.error("Error when procesing request body: #{inspect(reason)}") - # Maybe delete the batch + {batch_changeset, proofs} = + batch + |> Utils.extract_info_from_data_pointer() + |> Batches.generate_changesets() + + Batches.insert_or_update(batch_changeset, proofs) + |> case do + {:ok, _} -> + PubSub.broadcast(Explorer.PubSub, "update_views", %{ + eth_usd: + case EthConverter.get_eth_price_usd() do + {:ok, eth_usd_price} -> eth_usd_price + {:error, _error} -> :empty + end + }) + + {:error, error} -> + Logger.error( + "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" + ) + + # no changes in DB + nil -> + nil end + + "Done processing batch: #{batch.merkle_root}" |> Logger.debug() + Mutex.release(BatchMutex, lock) end end From ae13563ebbaeee9c09aff511b231bc915a9f6df3 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:20:49 -0300 Subject: [PATCH 04/30] fix: adding missing changes --- explorer/lib/explorer_web/live/utils.ex | 59 +++++++++++++++---------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index a31033bb2..375322383 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -1,7 +1,5 @@ # Frontend Utils defmodule ExplorerWeb.Helpers do - @request_timeout 10_000 - def shorten_hash(hash, decimals \\ 6) do case String.length(hash) do n when n < decimals -> hash @@ -134,6 +132,8 @@ end # Backend utils defmodule Utils do require Logger + # 256 KB + @max_body_size 256 * 1024 def string_to_bytes32(hex_string) do # Remove the '0x' prefix @@ -170,21 +170,40 @@ defmodule Utils do end def calculate_proof_hashes({:ok, deserialized_batch}) do - proof_hashes = - deserialized_batch - |> Enum.map(fn s3_object -> - :crypto.hash(:sha3_256, s3_object["proof"]) - end) + deserialized_batch + |> Enum.map(fn s3_object -> + :crypto.hash(:sha3_256, s3_object["proof"]) + end) + end + + def calculate_proof_hashes({:error, reason}) do + Logger.error("Error calculating proof hashes: #{inspect(reason)}") + [] + end + + defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} + defp stream_handler({:status, 200}, acc), do: {:cont, acc} - {:ok, proof_hashes} + defp stream_handler({:status, status_code}, _acc), + do: {:halt, {:error, {:http_error, status_code}}} + + defp stream_handler({:data, chunk}, {_acc_body, size}) + when size + byte_size(chunk) > @max_body_size do + {:halt, {:error, :body_too_large}} end - def calculate_proof_hashes(error), do: error + defp stream_handler({:data, chunk}, {acc_body, size}) do + new_size = size + byte_size(chunk) + {:cont, {acc_body <> chunk, new_size}} + end def fetch_batch_data_pointer(batch_data_pointer) do case Finch.build(:get, batch_data_pointer) - |> Finch.request(Explorer.Finch, request_timeout: @request_timeout) do - {:ok, %Finch.Response{status: 200, body: body}} -> + |> Finch.stream_while(Explorer.Finch, {"", 0}, &stream_handler(&1, &2)) do + {:ok, {:error, :body_too_large}} -> + {:error, {:http_error, :body_too_large}} + + {:ok, {body, _size}} -> cond do is_json?(body) -> case Jason.decode(body) do @@ -209,9 +228,6 @@ defmodule Utils do {:error, :unknown_format} end - {:ok, %Finch.Response{status: status_code}} -> - {:error, {:http_error, status_code}} - {:error, reason} -> {:error, {:http_error, reason}} end @@ -243,7 +259,7 @@ defmodule Utils do def extract_info_from_data_pointer(%BatchDB{} = batch) do Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") # only get from s3 if not already in DB - result = + proof_hashes = case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do nil -> Logger.debug("Fetching from S3") @@ -255,17 +271,12 @@ defmodule Utils do proof_hashes -> # already processed and stored the S3 data Logger.debug("Fetching from DB") - {:ok, proof_hashes} + proof_hashes end - with {:ok, proof_hashes} <- result do - batch_info = - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) - - {:ok, batch_info} - end + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) end def fetch_eigen_operator_metadata(url) do From 6f6405a48da64dba00763f0933e00d8b29ce9935 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:24:57 -0300 Subject: [PATCH 05/30] refactor: fixing formatting issues --- explorer/lib/explorer/periodically.ex | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index d1b54df38..423b3738e 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -16,10 +16,8 @@ defmodule Explorer.Periodically do one_second = 1000 seconds_in_an_hour = 60 * 60 - # every 12 seconds, once per block - :timer.send_interval(one_second * 12, :batches) - # every 1 hour - :timer.send_interval(one_second * seconds_in_an_hour, :restakings) + :timer.send_interval(one_second * 12, :batches) # every 12 seconds, once per block + :timer.send_interval(one_second * seconds_in_an_hour, :restakings) # every 1 hour end # Reads and process last blocks for operators and restaking changes @@ -47,7 +45,6 @@ defmodule Explorer.Periodically do run_every_n_iterations = 8 new_count = rem(count + 1, run_every_n_iterations) - if new_count == 0 do Task.start(&process_unverified_batches/0) end @@ -99,9 +96,7 @@ defmodule Explorer.Periodically do }) {:error, error} -> - Logger.error( - "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" - ) + Logger.error("Some error in DB operation, not broadcasting update_views: #{inspect(error)}") # no changes in DB nil -> From 62c2fa5d61eb01c985c27677a3603cf37098707d Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:34:21 -0300 Subject: [PATCH 06/30] refactor: improve error msg --- explorer/lib/explorer_web/live/utils.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 375322383..897adba18 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -189,7 +189,7 @@ defmodule Utils do defp stream_handler({:data, chunk}, {_acc_body, size}) when size + byte_size(chunk) > @max_body_size do - {:halt, {:error, :body_too_large}} + {:halt, {:error, {:http, :body_too_large}}} end defp stream_handler({:data, chunk}, {acc_body, size}) do @@ -200,8 +200,8 @@ defmodule Utils do def fetch_batch_data_pointer(batch_data_pointer) do case Finch.build(:get, batch_data_pointer) |> Finch.stream_while(Explorer.Finch, {"", 0}, &stream_handler(&1, &2)) do - {:ok, {:error, :body_too_large}} -> - {:error, {:http_error, :body_too_large}} + {:ok, {:error, reason}} -> + {:error, reason} {:ok, {body, _size}} -> cond do From d2519f043f7dc4719f88b0a2c3491a2bcc13905a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:48:08 -0300 Subject: [PATCH 07/30] refactor: change constant to env --- explorer/.env.dev | 2 ++ explorer/lib/explorer_web/live/utils.ex | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/explorer/.env.dev b/explorer/.env.dev index c93bcea2c..d594e549a 100644 --- a/explorer/.env.dev +++ b/explorer/.env.dev @@ -20,3 +20,5 @@ DEBUG_ERRORS=true # Operator version tracker API TRACKER_API_URL=http://localhost:3030 + +MAX_BATCH_SIZE=268435456 # 256 MiB diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 897adba18..4332850dd 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,8 +132,7 @@ end # Backend utils defmodule Utils do require Logger - # 256 KB - @max_body_size 256 * 1024 + @max_batch_size String.to_integer(System.get_env("POOL_SIZE") || "268435456") def string_to_bytes32(hex_string) do # Remove the '0x' prefix @@ -188,7 +187,7 @@ defmodule Utils do do: {:halt, {:error, {:http_error, status_code}}} defp stream_handler({:data, chunk}, {_acc_body, size}) - when size + byte_size(chunk) > @max_body_size do + when size + byte_size(chunk) > @max_batch_size do {:halt, {:error, {:http, :body_too_large}}} end From 951eb28172dfa44e54c1d00aa44c44cc488c1b4c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 16:00:23 -0300 Subject: [PATCH 08/30] fix: env variable --- explorer/lib/explorer_web/live/utils.ex | 2 +- explorer/start.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 4332850dd..9fabfd9ab 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,7 +132,7 @@ end # Backend utils defmodule Utils do require Logger - @max_batch_size String.to_integer(System.get_env("POOL_SIZE") || "268435456") + @max_batch_size String.to_integer(System.get_env("MAX_BATCH_SIZE") || "268435456") def string_to_bytes32(hex_string) do # Remove the '0x' prefix diff --git a/explorer/start.sh b/explorer/start.sh index 02d76fa6a..94786e011 100755 --- a/explorer/start.sh +++ b/explorer/start.sh @@ -14,6 +14,7 @@ env_vars=( "ALIGNED_CONFIG_FILE" "DEBUG_ERRORS" "TRACKER_API_URL" + "MAX_BATCH_SIZE" ) for var in "${env_vars[@]}"; do From 44552b3444239a0d115ec39e491fcc17e62da50b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 16:28:50 -0300 Subject: [PATCH 09/30] refactor: improve error handling --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 9fabfd9ab..19e765679 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -177,7 +177,7 @@ defmodule Utils do def calculate_proof_hashes({:error, reason}) do Logger.error("Error calculating proof hashes: #{inspect(reason)}") - [] + ["invalid"] end defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} From 01c5b2751753db88d37f002b9caf29d812d59691 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 16:38:11 -0300 Subject: [PATCH 10/30] test: add exploit test --- go.mod | 3 +- operator/pkg/operator_test.go | 249 ++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 operator/pkg/operator_test.go diff --git a/go.mod b/go.mod index 502e791b0..ba6808428 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/consensys/gnark-crypto v0.12.2-0.20240215234832-d72fcb379d3e github.com/fxamacker/cbor/v2 v2.7.0 github.com/ugorji/go/codec v1.2.12 + golang.org/x/net v0.24.0 ) require ( @@ -84,9 +85,9 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.24.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.20.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/operator/pkg/operator_test.go b/operator/pkg/operator_test.go new file mode 100644 index 000000000..090a8d39d --- /dev/null +++ b/operator/pkg/operator_test.go @@ -0,0 +1,249 @@ +package operator_test + +import ( + "context" + "crypto/ecdsa" + "crypto/rand" + "fmt" + "math/big" + "net/http" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + gethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/params" + contractAlignedLayerServiceManager "github.com/yetanotherco/aligned_layer/contracts/bindings/AlignedLayerServiceManager" + "golang.org/x/net/http2" +) + +var ( + alicePrivateKey = func() []byte { + key, err := hexutil.Decode("0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80") + if err != nil { + panic(err) + } + return key + }() + aliceAddress = func() common.Address { + addr, err := common.NewMixedcaseAddressFromString("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266") + if err != nil { + panic(err) + } + return addr.Address() + }() + alignedLayerServiceManagerAddress = func() common.Address { + addr, err := common.NewMixedcaseAddressFromString("0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8") + if err != nil { + panic(err) + } + return addr.Address() + }() +) + +func NewAnvilClient(t *testing.T) *ethclient.Client { + client, err := ethclient.Dial("http://localhost:8545") + if err != nil { + t.Fatalf("could not connect to anvil: %s", err) + } + return client +} + +type AlignedUser struct { + Signer gethtypes.Signer + Client *ethclient.Client + PrivateKey *ecdsa.PrivateKey + UserAddress common.Address + Name string +} + +func NewAlignedUser(t *testing.T, privateKey []byte, client *ethclient.Client, addr common.Address, name string) *AlignedUser { + key, err := crypto.ToECDSA(privateKey) + if err != nil { + t.Fatalf("could not create private key: %s", err) + } + return &AlignedUser{ + Signer: gethtypes.NewCancunSigner(big.NewInt(31337)), + Client: client, + PrivateKey: key, + UserAddress: addr, + Name: name, + } +} + +func (u *AlignedUser) SendTransaction(tx *gethtypes.Transaction) error { + signedTx, err := gethtypes.SignTx(tx, u.Signer, u.PrivateKey) + if err != nil { + return err + } + return u.Client.SendTransaction(context.TODO(), signedTx) +} + +func (u *AlignedUser) getNonce() *big.Int { + nonce, err := u.Client.NonceAt(context.Background(), u.UserAddress, nil) + if err != nil { + panic(err) + } + return new(big.Int).SetUint64(nonce) +} + +func CreateNewTask(t *testing.T, user *AlignedUser, contractAddress common.Address, merkleRoot [32]byte, dataPointer string) { + serviceManager, err := contractAlignedLayerServiceManager.NewContractAlignedLayerServiceManager( + contractAddress, + user.Client, + ) + if err != nil { + t.Fatalf("could not create service manager: %s", err) + } + t.Logf("ServiceManager created") + + DepositToUser(t, user, serviceManager) + + createTx, err := serviceManager.CreateNewTask( + &bind.TransactOpts{ + From: user.UserAddress, + Nonce: user.getNonce(), + Signer: func(addr common.Address, tx *gethtypes.Transaction) (*gethtypes.Transaction, error) { + return gethtypes.SignTx(tx, user.Signer, user.PrivateKey) + }, + Value: new(big.Int).SetUint64(1), + GasLimit: params.GenesisGasLimit / 2, + }, + merkleRoot, + dataPointer, + big.NewInt(1510000000000000), + ) + if err != nil { + t.Fatalf("could not create task: %s", err) + } + + t.Logf("New task created") + + i := 0 + r := new(gethtypes.Receipt) + for { + r, err = user.Client.TransactionReceipt(context.TODO(), createTx.Hash()) + if i > 10 { + return + } + i++ + if err != nil { + if err.Error() != "not found" { + t.Fatal(err) + } + time.Sleep(1 * time.Second) + continue + } + t.Logf("Receipt Status: %v", r.Status) + if r.Status != 0 { + break + } + time.Sleep(1 * time.Second) + } + +} + +func DepositToUser(t *testing.T, user *AlignedUser, serviceManager *contractAlignedLayerServiceManager.ContractAlignedLayerServiceManager) { + depositTx, err := serviceManager.DepositToBatcher( + &bind.TransactOpts{ + From: user.UserAddress, + Nonce: user.getNonce(), + Signer: func(addr common.Address, tx *gethtypes.Transaction) (*gethtypes.Transaction, error) { + return gethtypes.SignTx(tx, user.Signer, user.PrivateKey) + }, + Value: new(big.Int).SetUint64(2000000000000000), + GasLimit: params.GenesisGasLimit / 2, + }, + user.UserAddress, + ) + if err != nil { + t.Fatalf("could not create task: %s", err) + } + + t.Logf("New task created") + + i := 0 + r := new(gethtypes.Receipt) + for { + r, err = user.Client.TransactionReceipt(context.TODO(), depositTx.Hash()) + if i > 10 { + return + } + i++ + if err != nil { + if err.Error() != "not found" { + t.Fatal(err) + } + time.Sleep(1 * time.Second) + continue + } + t.Logf("Receipt Status: %v", r.Status) + if r.Status != 0 { + break + } + time.Sleep(1 * time.Second) + } + +} + +func processGzipBomb(t *testing.T) { + client := NewAnvilClient(t) + alice := NewAlignedUser(t, alicePrivateKey, client, aliceAddress, "alice") + t.Logf("New user %v message", alice) + + var randHash [32]byte + if _, err := rand.Read(randHash[:]); err != nil { + t.Fatalf("could not generate random hash: %s", err) + } + + CreateNewTask(t, alice, alignedLayerServiceManagerAddress, randHash, fmt.Sprintf("http://localhost:1515/%x", randHash)) +} + +func startTestServerOOM(t *testing.T, wg *sync.WaitGroup) { + wg.Add(1) + defer wg.Done() + + server := &http.Server{ + Addr: ":1515", + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handleRequestOOM(t, w, r) + }), + } + + if err := http2.ConfigureServer(server, &http2.Server{}); err != nil { + t.Fatalf("could not configure server for http2: %s", err) + } + t.Logf("Starting the server") + + t.Fatal(server.ListenAndServe()) +} + +func handleRequestOOM(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Logf("Received request: %s %s", r.Method, r.URL) + switch r.Method { + case http.MethodHead: + case http.MethodGet: + w.Header().Set("Content-Encoding", "gzip") + for { + if _, err := w.Write([]byte("infinite content")); err != nil { + t.Logf("Finishing test") + return + } + w.(http.Flusher).Flush() + } + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } +} + +func TestGetBatchExplorerOOM(t *testing.T) { + wg := sync.WaitGroup{} + go startTestServerOOM(t, &wg) + processGzipBomb(t) + wg.Wait() +} From 3bbd2a649617eaf1f6e6b87f72bba5daa179409c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 16:48:02 -0300 Subject: [PATCH 11/30] refactor: improve lgos --- operator/pkg/operator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/pkg/operator_test.go b/operator/pkg/operator_test.go index 090a8d39d..c99927510 100644 --- a/operator/pkg/operator_test.go +++ b/operator/pkg/operator_test.go @@ -165,7 +165,7 @@ func DepositToUser(t *testing.T, user *AlignedUser, serviceManager *contractAlig t.Fatalf("could not create task: %s", err) } - t.Logf("New task created") + t.Logf("Deposit sent") i := 0 r := new(gethtypes.Receipt) @@ -231,7 +231,7 @@ func handleRequestOOM(t *testing.T, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Encoding", "gzip") for { if _, err := w.Write([]byte("infinite content")); err != nil { - t.Logf("Finishing test") + t.Logf("Request finished") return } w.(http.Flusher).Flush() From 88aaca73d55245aa1ff5ba3edb9daa4bdf39ba57 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 17:17:18 -0300 Subject: [PATCH 12/30] refactor: add comment --- explorer/lib/explorer_web/live/utils.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 19e765679..474353c62 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -177,6 +177,7 @@ defmodule Utils do def calculate_proof_hashes({:error, reason}) do Logger.error("Error calculating proof hashes: #{inspect(reason)}") + # This ensures we avoid attempting to fetch the invalid data again. ["invalid"] end From e54d15b4360af6ad2e0125358151d584463459fe Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 00:57:34 -0300 Subject: [PATCH 13/30] refactor: error message --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 474353c62..85c0ed3c9 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -178,7 +178,7 @@ defmodule Utils do def calculate_proof_hashes({:error, reason}) do Logger.error("Error calculating proof hashes: #{inspect(reason)}") # This ensures we avoid attempting to fetch the invalid data again. - ["invalid"] + ["invalid batch"] end defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} From f2f3fcd8632aeaa08a1888eb1e9d338657ff2501 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:00:54 -0300 Subject: [PATCH 14/30] refactor: better error handling --- explorer/lib/explorer_web/live/utils.ex | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 85c0ed3c9..070794d42 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -175,12 +175,6 @@ defmodule Utils do end) end - def calculate_proof_hashes({:error, reason}) do - Logger.error("Error calculating proof hashes: #{inspect(reason)}") - # This ensures we avoid attempting to fetch the invalid data again. - ["invalid batch"] - end - defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} defp stream_handler({:status, 200}, acc), do: {:cont, acc} @@ -264,9 +258,17 @@ defmodule Utils do nil -> Logger.debug("Fetching from S3") - batch.data_pointer - |> Utils.fetch_batch_data_pointer() - |> Utils.calculate_proof_hashes() + batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() + case batch_content do + {:ok, batch_content} -> + batch_content + |> Utils.calculate_proof_hashes() + + {:error, reason} -> + Logger.error("Error fetching batch content: #{inspect(reason)}") + # Returning something ensures we avoid attempting to fetch the invalid data again. + ["invalid batch"] + end proof_hashes -> # already processed and stored the S3 data From e69bc14bcae28a21d9c682565b0458df0ade2948 Mon Sep 17 00:00:00 2001 From: Uriel Mihura <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:08:57 -0300 Subject: [PATCH 15/30] Update explorer/lib/explorer_web/live/utils.ex --- explorer/lib/explorer_web/live/utils.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 070794d42..f4d140e2d 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,7 +132,11 @@ end # Backend utils defmodule Utils do require Logger - @max_batch_size String.to_integer(System.get_env("MAX_BATCH_SIZE") || "268435456") + @max_batch_size (case System.fetch_env("MAX_BATCH_SIZE") do + {:ok, ""} -> 268_435_456 #empty env var + {:ok, value} -> String.to_integer(value) + _ -> 268_435_456 # error + end) def string_to_bytes32(hex_string) do # Remove the '0x' prefix From 3875037af26b5a450d0a78f83d877edfbe44c623 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:13:31 -0300 Subject: [PATCH 16/30] fix: calculate_proof_hashes --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index f4d140e2d..4d0cf24ef 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -172,7 +172,7 @@ defmodule Utils do |> Enum.reverse() end - def calculate_proof_hashes({:ok, deserialized_batch}) do + def calculate_proof_hashes(deserialized_batch) do deserialized_batch |> Enum.map(fn s3_object -> :crypto.hash(:sha3_256, s3_object["proof"]) From 5cee221b884a13b3a9124d101b6fb8a2e7da1bee Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:31:39 -0300 Subject: [PATCH 17/30] refactor: check content lenght in the headers --- explorer/lib/explorer_web/live/utils.ex | 33 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 4d0cf24ef..c9fdba88d 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,11 +132,14 @@ end # Backend utils defmodule Utils do require Logger + @max_batch_size (case System.fetch_env("MAX_BATCH_SIZE") do - {:ok, ""} -> 268_435_456 #empty env var - {:ok, value} -> String.to_integer(value) - _ -> 268_435_456 # error - end) + # empty env var + {:ok, ""} -> 268_435_456 + {:ok, value} -> String.to_integer(value) + # error + _ -> 268_435_456 + end) def string_to_bytes32(hex_string) do # Remove the '0x' prefix @@ -179,20 +182,27 @@ defmodule Utils do end) end - defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} + defp stream_handler({:headers, headers}, acc) do + {_, batch_size} = List.keyfind(headers, "content-length", 0) + check_batch_size(batch_size, acc) + end + defp stream_handler({:status, 200}, acc), do: {:cont, acc} defp stream_handler({:status, status_code}, _acc), do: {:halt, {:error, {:http_error, status_code}}} - defp stream_handler({:data, chunk}, {_acc_body, size}) - when size + byte_size(chunk) > @max_batch_size do - {:halt, {:error, {:http, :body_too_large}}} + defp stream_handler({:data, chunk}, {acc_body, acc_size}) do + new_size = acc_size + byte_size(chunk) + check_batch_size(new_size, {acc_body <> chunk, new_size}) end - defp stream_handler({:data, chunk}, {acc_body, size}) do - new_size = size + byte_size(chunk) - {:cont, {acc_body <> chunk, new_size}} + defp check_batch_size(size, acc) do + if size > @max_batch_size do + {:halt, {:error, {:http, :body_too_large}}} + else + {:cont, acc} + end end def fetch_batch_data_pointer(batch_data_pointer) do @@ -263,6 +273,7 @@ defmodule Utils do Logger.debug("Fetching from S3") batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() + case batch_content do {:ok, batch_content} -> batch_content From 9c5ef15677f8cf047cf4455c924415eed4834c9b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:35:44 -0300 Subject: [PATCH 18/30] fix: set invalid proof hashes to 0 --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index c9fdba88d..694e35d22 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -282,7 +282,7 @@ defmodule Utils do {:error, reason} -> Logger.error("Error fetching batch content: #{inspect(reason)}") # Returning something ensures we avoid attempting to fetch the invalid data again. - ["invalid batch"] + ["0"] end proof_hashes -> From 8ab0e938cc1a765e6e2519438a9d8b5bc75bc2c0 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:46:49 -0300 Subject: [PATCH 19/30] fix: use a binary zero --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 694e35d22..8aeee8cfa 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -282,7 +282,7 @@ defmodule Utils do {:error, reason} -> Logger.error("Error fetching batch content: #{inspect(reason)}") # Returning something ensures we avoid attempting to fetch the invalid data again. - ["0"] + [<<0>>] end proof_hashes -> From 0466b3b04537c266a92a230512b6a237c683971b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:59:25 -0300 Subject: [PATCH 20/30] fix: set a default content length --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 8aeee8cfa..d4e5a0809 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -183,7 +183,7 @@ defmodule Utils do end defp stream_handler({:headers, headers}, acc) do - {_, batch_size} = List.keyfind(headers, "content-length", 0) + {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, 0}) check_batch_size(batch_size, acc) end From a735611404287633a482d1bd45e34316f4f32658 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 10 Oct 2024 10:48:52 -0300 Subject: [PATCH 21/30] feat(explorer): add is_valid field to batches --- explorer/lib/explorer/models/batch_structs.ex | 6 +- explorer/lib/explorer/models/batches.ex | 10 ++-- explorer/lib/explorer_web/live/utils.ex | 57 ++++++++++++------- .../20241010133259_add_is_valid_field.exs | 9 +++ 4 files changed, 54 insertions(+), 28 deletions(-) create mode 100644 explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs diff --git a/explorer/lib/explorer/models/batch_structs.ex b/explorer/lib/explorer/models/batch_structs.ex index 0ba1329d1..599c0f0d1 100644 --- a/explorer/lib/explorer/models/batch_structs.ex +++ b/explorer/lib/explorer/models/batch_structs.ex @@ -32,7 +32,8 @@ defmodule BatchDB do :proof_hashes, :fee_per_proof, :sender_address, - :max_aggregator_fee + :max_aggregator_fee, + :is_valid ] defstruct [ :merkle_root, @@ -48,6 +49,7 @@ defmodule BatchDB do :proof_hashes, :fee_per_proof, :sender_address, - :max_aggregator_fee + :max_aggregator_fee, + :is_valid ] end diff --git a/explorer/lib/explorer/models/batches.ex b/explorer/lib/explorer/models/batches.ex index 7d2b430af..e3700b7c6 100644 --- a/explorer/lib/explorer/models/batches.ex +++ b/explorer/lib/explorer/models/batches.ex @@ -18,6 +18,7 @@ defmodule Batches do field :fee_per_proof, :integer field :sender_address, :binary field :max_aggregator_fee, :decimal + field :is_valid, :boolean timestamps() end @@ -25,8 +26,8 @@ defmodule Batches do @doc false def changeset(new_batch, updates) do new_batch - |> cast(updates, [:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :submission_timestamp, :response_block_number, :response_transaction_hash, :response_timestamp, :data_pointer, :fee_per_proof, :sender_address, :max_aggregator_fee]) - |> validate_required([:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :fee_per_proof, :sender_address]) + |> cast(updates, [:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :submission_timestamp, :response_block_number, :response_transaction_hash, :response_timestamp, :data_pointer, :fee_per_proof, :sender_address, :max_aggregator_fee, :is_valid]) + |> validate_required([:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :fee_per_proof, :sender_address, :is_valid]) |> validate_format(:merkle_root, ~r/0x[a-fA-F0-9]{64}/) |> unique_constraint(:merkle_root) |> validate_number(:amount_of_proofs, greater_than: 0) @@ -37,6 +38,7 @@ defmodule Batches do |> validate_format(:response_transaction_hash, ~r/0x[a-fA-F0-9]{64}/) |> validate_number(:max_aggregator_fee, greater_than: 0) |> validate_number(:fee_per_proof, greater_than_or_equal_to: 0) + |> validate_inclusion(:is_verified, [true, false]) end def cast_to_batches(%BatchDB{} = batch_db) do @@ -53,7 +55,8 @@ defmodule Batches do data_pointer: batch_db.data_pointer, fee_per_proof: batch_db.fee_per_proof, sender_address: batch_db.sender_address, - max_aggregator_fee: batch_db.max_aggregator_fee + max_aggregator_fee: batch_db.max_aggregator_fee, + is_valid: batch_db.is_valid } end @@ -193,5 +196,4 @@ defmodule Batches do end end end - end diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index d4e5a0809..f4b19c387 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -264,36 +264,49 @@ defmodule Utils do end end - def extract_info_from_data_pointer(%BatchDB{} = batch) do - Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") + defp get_proof_hashes(%BatchDB{} = batch) do # only get from s3 if not already in DB - proof_hashes = - case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do - nil -> - Logger.debug("Fetching from S3") + case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do + nil -> + Logger.debug("Fetching from S3") - batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() + batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() - case batch_content do - {:ok, batch_content} -> + case batch_content do + {:ok, batch_content} -> + proof_hashes = batch_content |> Utils.calculate_proof_hashes() - {:error, reason} -> - Logger.error("Error fetching batch content: #{inspect(reason)}") - # Returning something ensures we avoid attempting to fetch the invalid data again. - [<<0>>] - end + {:ok, proof_hashes} - proof_hashes -> - # already processed and stored the S3 data - Logger.debug("Fetching from DB") - proof_hashes - end + {:error, reason} -> + Logger.error("Error fetching batch content: #{inspect(reason)}") + # Returning something ensures we avoid attempting to fetch the invalid data again. + {:error, [<<0>>]} + end + + proof_hashes -> + # already processed and stored the S3 data + Logger.debug("Fetching from DB") + {:ok, proof_hashes} + end + end + + def extract_info_from_data_pointer(%BatchDB{} = batch) do + Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) + {status, proof_hashes} = get_proof_hashes(batch) + + updated_batch = + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, Enum.count(proof_hashes)) + + case status do + :error -> Map.put(updated_batch, :is_valid, false) + _ -> updated_batch + end end def fetch_eigen_operator_metadata(url) do diff --git a/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs b/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs new file mode 100644 index 000000000..f229471a9 --- /dev/null +++ b/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs @@ -0,0 +1,9 @@ +defmodule Explorer.Repo.Migrations.AddIsValidField do + use Ecto.Migration + + def change do + alter table("batches") do + add :is_valid, :boolean + end + end +end From abf078e9da75eaaa7410055c05ff3ed35f1aa542 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 10 Oct 2024 10:53:16 -0300 Subject: [PATCH 22/30] fix: change is_verified to is_valid --- explorer/lib/explorer/models/batches.ex | 2 +- explorer/lib/explorer_web/live/utils.ex | 32 ++++++++++++------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/explorer/lib/explorer/models/batches.ex b/explorer/lib/explorer/models/batches.ex index e3700b7c6..afd4f4121 100644 --- a/explorer/lib/explorer/models/batches.ex +++ b/explorer/lib/explorer/models/batches.ex @@ -38,7 +38,7 @@ defmodule Batches do |> validate_format(:response_transaction_hash, ~r/0x[a-fA-F0-9]{64}/) |> validate_number(:max_aggregator_fee, greater_than: 0) |> validate_number(:fee_per_proof, greater_than_or_equal_to: 0) - |> validate_inclusion(:is_verified, [true, false]) + |> validate_inclusion(:is_valid, [true, false]) end def cast_to_batches(%BatchDB{} = batch_db) do diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index f4b19c387..2af93a6ec 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -264,6 +264,22 @@ defmodule Utils do end end + def extract_info_from_data_pointer(%BatchDB{} = batch) do + Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") + + {status, proof_hashes} = get_proof_hashes(batch) + + updated_batch = + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, Enum.count(proof_hashes)) + + case status do + :error -> Map.put(updated_batch, :is_valid, false) + _ -> updated_batch + end + end + defp get_proof_hashes(%BatchDB{} = batch) do # only get from s3 if not already in DB case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do @@ -293,22 +309,6 @@ defmodule Utils do end end - def extract_info_from_data_pointer(%BatchDB{} = batch) do - Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") - - {status, proof_hashes} = get_proof_hashes(batch) - - updated_batch = - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, Enum.count(proof_hashes)) - - case status do - :error -> Map.put(updated_batch, :is_valid, false) - _ -> updated_batch - end - end - def fetch_eigen_operator_metadata(url) do case Finch.build(:get, url) |> Finch.request(Explorer.Finch) do {:ok, %Finch.Response{status: 200, body: body}} -> From 48c61df74e611c1b64dde4bd083fa3fea44486a4 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 10 Oct 2024 12:01:04 -0300 Subject: [PATCH 23/30] fix: set true as a default for is_valid field --- explorer/lib/explorer/models/batches.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer/models/batches.ex b/explorer/lib/explorer/models/batches.ex index afd4f4121..bef38a130 100644 --- a/explorer/lib/explorer/models/batches.ex +++ b/explorer/lib/explorer/models/batches.ex @@ -18,7 +18,7 @@ defmodule Batches do field :fee_per_proof, :integer field :sender_address, :binary field :max_aggregator_fee, :decimal - field :is_valid, :boolean + field :is_valid, :boolean, default: true timestamps() end From f6d3cea22a6bd99aba92b2db38974aa405e414f0 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 10 Oct 2024 17:03:00 -0300 Subject: [PATCH 24/30] feat: add is valid status to frontend --- .../aligned_layer_service_manager.ex | 6 ++- explorer/lib/explorer/models/batches.ex | 2 +- .../components/core_components.ex | 41 +++++++++++++++++-- .../live/pages/batch/index.html.heex | 2 +- .../live/pages/batches/index.html.heex | 2 +- .../live/pages/operators/index.ex | 2 +- explorer/lib/explorer_web/live/utils.ex | 8 ++++ .../20241010133259_add_is_valid_field.exs | 2 +- 8 files changed, 54 insertions(+), 11 deletions(-) diff --git a/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex b/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex index fb31b19d5..dca61d58b 100644 --- a/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex +++ b/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex @@ -136,7 +136,8 @@ defmodule AlignedLayerServiceManager do proof_hashes: nil, fee_per_proof: BatcherPaymentServiceManager.get_fee_per_proof(%{merkle_root: created_batch.batchMerkleRoot}), sender_address: Utils.string_to_bytes32(created_batch.senderAddress), - max_aggregator_fee: created_batch.maxAggregatorFee + max_aggregator_fee: created_batch.maxAggregatorFee, + is_valid: true } end @@ -166,7 +167,8 @@ defmodule AlignedLayerServiceManager do fee_per_proof: unverified_batch.fee_per_proof, proof_hashes: nil, sender_address: unverified_batch.sender_address, - max_aggregator_fee: unverified_batch.max_aggregator_fee + max_aggregator_fee: unverified_batch.max_aggregator_fee, + is_valid: true } end end diff --git a/explorer/lib/explorer/models/batches.ex b/explorer/lib/explorer/models/batches.ex index bef38a130..f2474ef48 100644 --- a/explorer/lib/explorer/models/batches.ex +++ b/explorer/lib/explorer/models/batches.ex @@ -116,7 +116,7 @@ defmodule Batches do threshold_datetime = DateTime.utc_now() |> DateTime.add(-43200, :second) # 12 hours ago query = from(b in Batches, - where: b.is_verified == false and b.submission_timestamp > ^threshold_datetime, + where: b.is_valid == true and b.is_verified == false and b.submission_timestamp > ^threshold_datetime, select: b) Explorer.Repo.all(query) diff --git a/explorer/lib/explorer_web/components/core_components.ex b/explorer/lib/explorer_web/components/core_components.ex index 54976b84f..02a49e565 100644 --- a/explorer/lib/explorer_web/components/core_components.ex +++ b/explorer/lib/explorer_web/components/core_components.ex @@ -417,15 +417,15 @@ defmodule ExplorerWeb.CoreComponents do end @doc """ - Renders a dynamic badge compoent. + Renders a dynamic badge component. """ attr :class, :string, default: nil attr :status, :boolean, default: true - attr :falsy_text, :string, default: "Pending" - attr :truthy_text, :string, default: "Verified" + attr :falsy_text, :string + attr :truthy_text, :string slot :inner_block, default: nil - def dynamic_badge(assigns) do + def dynamic_badge_boolean(assigns) do ~H""" <.badge variant={ @@ -449,6 +449,39 @@ defmodule ExplorerWeb.CoreComponents do """ end + @doc """ + Renders a dynamic badge component for the batcher. + """ + attr :class, :string, default: nil + attr :status, :atom + slot :inner_block, default: nil + + def dynamic_badge_for_batcher(assigns) do + ~H""" + <.badge + variant={ + case @status do + :invalid -> "destructive" + :verified -> "accent" + :pending -> "foreground" + end + } + class={ + classes([ + @class + ]) + } + > + <%= case @status do + :invalid -> "Invalid" + :verified -> "Verified" + :pending -> "Pending" + end %> + <%= render_slot(@inner_block) %> + + """ + end + @doc """ Renders an input with label and error messages. diff --git a/explorer/lib/explorer_web/live/pages/batch/index.html.heex b/explorer/lib/explorer_web/live/pages/batch/index.html.heex index 98636fec5..764f12219 100644 --- a/explorer/lib/explorer_web/live/pages/batch/index.html.heex +++ b/explorer/lib/explorer_web/live/pages/batch/index.html.heex @@ -25,7 +25,7 @@

Status:

- <.dynamic_badge class="w-fit" status={@current_batch.is_verified} /> + <.dynamic_badge_for_batcher class="w-fit" status={Helpers.get_batch_status(@current_batch)} />

diff --git a/explorer/lib/explorer_web/live/pages/batches/index.html.heex b/explorer/lib/explorer_web/live/pages/batches/index.html.heex index 044e43a14..26ce5ac2e 100644 --- a/explorer/lib/explorer_web/live/pages/batches/index.html.heex +++ b/explorer/lib/explorer_web/live/pages/batches/index.html.heex @@ -14,7 +14,7 @@ <:col :let={batch} label="Status"> - <.dynamic_badge status={batch.is_verified} /> + <.dynamic_badge_for_batcher status={Helpers.get_batch_status(batch)} /> <:col :let={batch} label="Age"> diff --git a/explorer/lib/explorer_web/live/pages/operators/index.ex b/explorer/lib/explorer_web/live/pages/operators/index.ex index 563cfce07..dab94a92e 100644 --- a/explorer/lib/explorer_web/live/pages/operators/index.ex +++ b/explorer/lib/explorer_web/live/pages/operators/index.ex @@ -83,7 +83,7 @@ defmodule ExplorerWeb.Operators.Index do <%= operator.total_stake |> EthConverter.wei_to_eth(2) |> Helpers.format_number() %> ETH <:col :let={operator} label="Status"> - <.dynamic_badge status={operator.is_active} truthy_text="Active" falsy_text="Inactive" /> + <.dynamic_badge_boolean status={operator.is_active} truthy_text="Active" falsy_text="Inactive" /> <% else %> diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 2af93a6ec..8d9be81ed 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -127,6 +127,14 @@ defmodule ExplorerWeb.Helpers do def binary_to_hex_string(binary) do Utils.binary_to_hex_string(binary) end + + def get_batch_status(batch) do + cond do + not batch.is_valid -> :invalid + batch.is_verified -> :verified + true -> :pending + end + end end # Backend utils diff --git a/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs b/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs index f229471a9..086431335 100644 --- a/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs +++ b/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs @@ -3,7 +3,7 @@ defmodule Explorer.Repo.Migrations.AddIsValidField do def change do alter table("batches") do - add :is_valid, :boolean + add :is_valid, :boolean, default: true end end end From bcc9a52c8b4b123d85bc9b30d63032e0ad2d9b23 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 10 Oct 2024 17:39:17 -0300 Subject: [PATCH 25/30] fix: convert content-length to integer --- explorer/lib/explorer_web/live/utils.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index d4e5a0809..048e36381 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -183,8 +183,8 @@ defmodule Utils do end defp stream_handler({:headers, headers}, acc) do - {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, 0}) - check_batch_size(batch_size, acc) + {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, "0"}) + check_batch_size(String.to_integer(batch_size), acc) end defp stream_handler({:status, 200}, acc), do: {:cont, acc} From 824fd46316893b5d66c2271a7764348e6deefe1f Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:59:25 -0300 Subject: [PATCH 26/30] fix: set a default content length --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 8aeee8cfa..d4e5a0809 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -183,7 +183,7 @@ defmodule Utils do end defp stream_handler({:headers, headers}, acc) do - {_, batch_size} = List.keyfind(headers, "content-length", 0) + {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, 0}) check_batch_size(batch_size, acc) end From 00a84949e71eb31321df6fa721a431f7d2ddff7b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 10 Oct 2024 17:39:17 -0300 Subject: [PATCH 27/30] fix: convert content-length to integer --- explorer/lib/explorer_web/live/utils.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index d4e5a0809..048e36381 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -183,8 +183,8 @@ defmodule Utils do end defp stream_handler({:headers, headers}, acc) do - {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, 0}) - check_batch_size(batch_size, acc) + {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, "0"}) + check_batch_size(String.to_integer(batch_size), acc) end defp stream_handler({:status, 200}, acc), do: {:cont, acc} From 6534a09b7aa09026aba7e45206a2bfb425383079 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Sat, 12 Oct 2024 17:20:11 -0300 Subject: [PATCH 28/30] refactor: improve error handling --- explorer/lib/explorer/periodically.ex | 36 +++++++++-------------- explorer/lib/explorer_web/live/utils.ex | 38 +++++++++++++++---------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index 423b3738e..973150244 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -45,6 +45,7 @@ defmodule Explorer.Periodically do run_every_n_iterations = 8 new_count = rem(count + 1, run_every_n_iterations) + if new_count == 0 do Task.start(&process_unverified_batches/0) end @@ -79,28 +80,19 @@ defmodule Explorer.Periodically do {:ok, lock} -> "Processing batch: #{batch.merkle_root}" |> Logger.debug() - {batch_changeset, proofs} = - batch - |> Utils.extract_info_from_data_pointer() - |> Batches.generate_changesets() - - Batches.insert_or_update(batch_changeset, proofs) - |> case do - {:ok, _} -> - PubSub.broadcast(Explorer.PubSub, "update_views", %{ - eth_usd: - case EthConverter.get_eth_price_usd() do - {:ok, eth_usd_price} -> eth_usd_price - {:error, _error} -> :empty - end - }) - - {:error, error} -> - Logger.error("Some error in DB operation, not broadcasting update_views: #{inspect(error)}") - - # no changes in DB - nil -> - nil + with {:ok, updated_batch} <- Utils.process_batch(batch), + {batch_changeset, proofs} <- Batches.generate_changesets(updated_batch), + {:ok, _} <- Batches.insert_or_update(batch_changeset, proofs) do + PubSub.broadcast(Explorer.PubSub, "update_views", %{ + eth_usd: + case EthConverter.get_eth_price_usd() do + {:ok, eth_usd_price} -> eth_usd_price + {:error, _error} -> :empty + end + }) + else + {:error, reason} -> + Logger.error("Error processing batch #{batch.merkle_root}. Error: #{inspect(reason)}") end "Done processing batch: #{batch.merkle_root}" |> Logger.debug() diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 17bf946c2..772fd507d 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -207,7 +207,7 @@ defmodule Utils do defp check_batch_size(size, acc) do if size > @max_batch_size do - {:halt, {:error, {:http, :body_too_large}}} + {:halt, {:error, {:invalid, body_too_large}}} else {:cont, acc} end @@ -241,7 +241,7 @@ defmodule Utils do true -> Logger.error("Unknown S3 object format") - {:error, :unknown_format} + {:error, {:invalid, :unknown_format}} end {:error, reason} -> @@ -272,23 +272,33 @@ defmodule Utils do end end - def extract_info_from_data_pointer(%BatchDB{} = batch) do - Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") + def process_batch(%BatchDB{} = batch) do + case get_proof_hashes(batch) do + {:ok, proof_hashes} -> + {:ok, add_proof_hashes_to_batch(batch, proof_hashes)} - {status, proof_hashes} = get_proof_hashes(batch) + {:error, {:invalid, reason}} -> + # Returning something ensures we avoid attempting to fetch the invalid data again. + updated_batch = + batch + |> Map.put(:is_valid, false) + |> add_proof_hashes_to_batch(<<0>>) - updated_batch = - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, Enum.count(proof_hashes)) + {:ok, updated_batch} - case status do - :error -> Map.put(updated_batch, :is_valid, false) - _ -> updated_batch + {:error, reason} -> + {:error, reason} end end + defp add_proof_hashes_to_batch(batch, proof_hashes) do + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, Enum.count(proof_hashes)) + end + defp get_proof_hashes(%BatchDB{} = batch) do + Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") # only get from s3 if not already in DB case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do nil -> @@ -305,9 +315,7 @@ defmodule Utils do {:ok, proof_hashes} {:error, reason} -> - Logger.error("Error fetching batch content: #{inspect(reason)}") - # Returning something ensures we avoid attempting to fetch the invalid data again. - {:error, [<<0>>]} + {:error, "Error fetching batch content: #{inspect(reason)}"} end proof_hashes -> From 35fe8864701f487eb920e42ebe45e22cb8f1c1f5 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Sat, 12 Oct 2024 17:32:18 -0300 Subject: [PATCH 29/30] fix: error handling --- explorer/lib/explorer_web/live/utils.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 772fd507d..68737af33 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -278,6 +278,7 @@ defmodule Utils do {:ok, add_proof_hashes_to_batch(batch, proof_hashes)} {:error, {:invalid, reason}} -> + Logger.error("Invalid batch content for #{batch.merkle_root}: #{inspect(reason)}") # Returning something ensures we avoid attempting to fetch the invalid data again. updated_batch = batch @@ -315,7 +316,7 @@ defmodule Utils do {:ok, proof_hashes} {:error, reason} -> - {:error, "Error fetching batch content: #{inspect(reason)}"} + {:error, reason} end proof_hashes -> From 0fa48b87241de77f2a33e5d5d97afe0b4df8df68 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Mon, 14 Oct 2024 10:26:59 -0300 Subject: [PATCH 30/30] fix: handle errors properly --- explorer/lib/explorer/periodically.ex | 3 +++ explorer/lib/explorer_web/live/utils.ex | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index 973150244..403d65140 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -93,6 +93,9 @@ defmodule Explorer.Periodically do else {:error, reason} -> Logger.error("Error processing batch #{batch.merkle_root}. Error: #{inspect(reason)}") + # no changes in DB + nil -> + nil end "Done processing batch: #{batch.merkle_root}" |> Logger.debug() diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 68737af33..270083825 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -207,7 +207,7 @@ defmodule Utils do defp check_batch_size(size, acc) do if size > @max_batch_size do - {:halt, {:error, {:invalid, body_too_large}}} + {:halt, {:error, {:invalid, :body_too_large}}} else {:cont, acc} end @@ -283,7 +283,7 @@ defmodule Utils do updated_batch = batch |> Map.put(:is_valid, false) - |> add_proof_hashes_to_batch(<<0>>) + |> add_proof_hashes_to_batch([<<0>>]) {:ok, updated_batch}