diff --git a/api/api.go b/api/api.go index f7af3f4c..0e98cfce 100644 --- a/api/api.go +++ b/api/api.go @@ -17,8 +17,9 @@ import ( "github.com/vocdoni/census3/db/annotations" queries "github.com/vocdoni/census3/db/sqlc" "github.com/vocdoni/census3/helpers/queue" + "github.com/vocdoni/census3/helpers/web3" "github.com/vocdoni/census3/scanner/providers" - "github.com/vocdoni/census3/scanner/providers/web3" + web3provider "github.com/vocdoni/census3/scanner/providers/web3" "go.vocdoni.io/dvote/api/censusdb" storagelayer "go.vocdoni.io/dvote/data" "go.vocdoni.io/dvote/data/downloader" @@ -247,7 +248,7 @@ func (capi *census3API) CreateInitialTokens(tokensPath string) error { continue } if !provider.IsExternal() { - if err := provider.SetRef(web3.Web3ProviderRef{ + if err := provider.SetRef(web3provider.Web3ProviderRef{ HexAddress: token.ID, ChainID: token.ChainID, }); err != nil { diff --git a/cmd/census3/main.go b/cmd/census3/main.go index 0014e9de..dece26f9 100644 --- a/cmd/census3/main.go +++ b/cmd/census3/main.go @@ -22,7 +22,8 @@ import ( gitcoinDB "github.com/vocdoni/census3/scanner/providers/gitcoin/db" "github.com/vocdoni/census3/scanner/providers/manager" "github.com/vocdoni/census3/scanner/providers/poap" - "github.com/vocdoni/census3/scanner/providers/web3" + web3provider "github.com/vocdoni/census3/scanner/providers/web3" + "github.com/vocdoni/census3/helpers/web3" "go.vocdoni.io/dvote/log" ) @@ -158,10 +159,10 @@ func main() { // init the provider manager pm := manager.NewProviderManager() // init the web3 token providers - web3ProviderConf := web3.Web3ProviderConfig{Endpoints: w3p} - pm.AddProvider(new(web3.ERC20HolderProvider).Type(), web3ProviderConf) - pm.AddProvider(new(web3.ERC721HolderProvider).Type(), web3ProviderConf) - pm.AddProvider(new(web3.ERC777HolderProvider).Type(), web3ProviderConf) + web3ProviderConf := web3provider.Web3ProviderConfig{Endpoints: w3p} + pm.AddProvider(new(web3provider.ERC20HolderProvider).Type(), web3ProviderConf) + pm.AddProvider(new(web3provider.ERC721HolderProvider).Type(), web3ProviderConf) + pm.AddProvider(new(web3provider.ERC777HolderProvider).Type(), web3ProviderConf) // init POAP external provider if config.poapAPIEndpoint != "" { pm.AddProvider(new(poap.POAPHolderProvider).Type(), poap.POAPConfig{ diff --git a/scanner/providers/web3/web3_client.go b/helpers/web3/web3_client.go similarity index 100% rename from scanner/providers/web3/web3_client.go rename to helpers/web3/web3_client.go diff --git a/scanner/providers/web3/web3_iter.go b/helpers/web3/web3_iter.go similarity index 100% rename from scanner/providers/web3/web3_iter.go rename to helpers/web3/web3_iter.go diff --git a/scanner/providers/web3/web3_pool.go b/helpers/web3/web3_pool.go similarity index 97% rename from scanner/providers/web3/web3_pool.go rename to helpers/web3/web3_pool.go index 94b38d04..0edd219d 100644 --- a/scanner/providers/web3/web3_pool.go +++ b/helpers/web3/web3_pool.go @@ -18,10 +18,18 @@ import ( "encoding/json" "fmt" "net/http" + "time" "github.com/ethereum/go-ethereum/ethclient" ) +const ( + DefaultMaxWeb3ClientRetries = 5 + + shortNameSourceUri = "https://chainid.network/chains_mini.json" + checkWeb3EndpointsTimeout = time.Second * 10 +) + // Web3Pool struct contains a map of chainID-[]*Web3Endpoint, where // the key is the chainID and the value is a list of Web3Endpoint. It also // contains a list of all the Web3Endpoint metadata. It provides methods to diff --git a/scanner/providers/farcaster/types.go b/scanner/providers/farcaster/types.go index 7c3040a6..0e23b31a 100644 --- a/scanner/providers/farcaster/types.go +++ b/scanner/providers/farcaster/types.go @@ -9,7 +9,7 @@ import ( "github.com/ethereum/go-ethereum/common" fcir "github.com/vocdoni/census3/contracts/farcaster/idRegistry" fckr "github.com/vocdoni/census3/contracts/farcaster/keyRegistry" - "github.com/vocdoni/census3/scanner/providers/web3" + "github.com/vocdoni/census3/helpers/web3" ) type FarcasterProviderConf struct { diff --git a/scanner/providers/web3/const.go b/scanner/providers/web3/const.go index 1b70efa4..6f7c36a7 100644 --- a/scanner/providers/web3/const.go +++ b/scanner/providers/web3/const.go @@ -1,19 +1,17 @@ package web3 -import "time" +import ( + "time" -const ( - DefaultMaxWeb3ClientRetries = 5 - RetryWeb3Cooldown = 500 * time.Millisecond + "github.com/vocdoni/census3/helpers/web3" ) const ( - shortNameSourceUri = "https://chainid.network/chains_mini.json" - checkWeb3EndpointsTimeout = time.Second * 10 - TimeLayout = "2006-01-02T15:04:05Z07:00" + RetryWeb3Cooldown = 500 * time.Millisecond + TimeLayout = "2006-01-02T15:04:05Z07:00" ) -var DefaultWeb3Endpoint = &Web3Endpoint{ +var DefaultWeb3Endpoint = &web3.Web3Endpoint{ ChainID: 11155111, Name: "Sepolia", ShortName: "sep", diff --git a/scanner/providers/web3/erc20_provider.go b/scanner/providers/web3/erc20_provider.go index 9b7db357..bfc9dd16 100644 --- a/scanner/providers/web3/erc20_provider.go +++ b/scanner/providers/web3/erc20_provider.go @@ -11,12 +11,13 @@ import ( "github.com/ethereum/go-ethereum/common" erc20 "github.com/vocdoni/census3/contracts/erc/erc20" "github.com/vocdoni/census3/scanner/providers" + "github.com/vocdoni/census3/helpers/web3" "go.vocdoni.io/dvote/log" ) type ERC20HolderProvider struct { - endpoints *Web3Pool - client *Client + endpoints *web3.Web3Pool + client *web3.Client contract *erc20.ERC20Contract address common.Address diff --git a/scanner/providers/web3/erc721_provider.go b/scanner/providers/web3/erc721_provider.go index 25bcb2a8..fc6acbec 100644 --- a/scanner/providers/web3/erc721_provider.go +++ b/scanner/providers/web3/erc721_provider.go @@ -11,12 +11,13 @@ import ( "github.com/ethereum/go-ethereum/common" erc721 "github.com/vocdoni/census3/contracts/erc/erc721" "github.com/vocdoni/census3/scanner/providers" + "github.com/vocdoni/census3/helpers/web3" "go.vocdoni.io/dvote/log" ) type ERC721HolderProvider struct { - endpoints *Web3Pool - client *Client + endpoints *web3.Web3Pool + client *web3.Client contract *erc721.ERC721Contract address common.Address diff --git a/scanner/providers/web3/erc777_provider.go b/scanner/providers/web3/erc777_provider.go index 94c74e2d..9eefd1c4 100644 --- a/scanner/providers/web3/erc777_provider.go +++ b/scanner/providers/web3/erc777_provider.go @@ -11,12 +11,13 @@ import ( "github.com/ethereum/go-ethereum/common" erc777 "github.com/vocdoni/census3/contracts/erc/erc777" "github.com/vocdoni/census3/scanner/providers" + "github.com/vocdoni/census3/helpers/web3" "go.vocdoni.io/dvote/log" ) type ERC777HolderProvider struct { - endpoints *Web3Pool - client *Client + endpoints *web3.Web3Pool + client *web3.Client contract *erc777.ERC777Contract address common.Address diff --git a/scanner/providers/web3/web3_provider.go b/scanner/providers/web3/web3_provider.go index f56509c1..b1aae0eb 100644 --- a/scanner/providers/web3/web3_provider.go +++ b/scanner/providers/web3/web3_provider.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/vocdoni/census3/scanner/providers" + "github.com/vocdoni/census3/helpers/web3" "go.vocdoni.io/dvote/db" "go.vocdoni.io/dvote/log" ) @@ -24,19 +25,19 @@ type Web3ProviderRef struct { type Web3ProviderConfig struct { Web3ProviderRef - Endpoints *Web3Pool + Endpoints *web3.Web3Pool DB *db.Database } // creationBlock function returns the block number of the creation of a contract // address. It uses the `eth_getCode` method to get the contract code at the // block number provided. If the method is not supported, it returns 0 and nil. -func creationBlock(client *Client, ctx context.Context, addr common.Address) (uint64, error) { +func creationBlock(client *web3.Client, ctx context.Context, addr common.Address) (uint64, error) { // check if the current client supports `eth_getCode` method, if not, return // 1 and nil. It is assumed that the contract is created at block 1 to start // scanning from the first block. getCodeSupport := false - for i := 0; i < DefaultMaxWeb3ClientRetries; i++ { + for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ { ethClient, err := client.EthClient() if err != nil { return 0, err @@ -52,7 +53,7 @@ func creationBlock(client *Client, ctx context.Context, addr common.Address) (ui // get the latest block number var err error var lastBlock uint64 - for i := 0; i < DefaultMaxWeb3ClientRetries; i++ { + for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ { lastBlock, err = client.BlockNumber(ctx) if err == nil { break @@ -63,7 +64,7 @@ func creationBlock(client *Client, ctx context.Context, addr common.Address) (ui return 0, err } var creationBlock uint64 - for i := 0; i < DefaultMaxWeb3ClientRetries; i++ { + for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ { creationBlock, err = creationBlockInRange(client, ctx, addr, 0, lastBlock) if err == nil { break @@ -75,7 +76,9 @@ func creationBlock(client *Client, ctx context.Context, addr common.Address) (ui // creationBlockInRange function finds the block number of a contract between // the bounds provided as start and end blocks. -func creationBlockInRange(client *Client, ctx context.Context, addr common.Address, start, end uint64) (uint64, error) { +func creationBlockInRange(client *web3.Client, ctx context.Context, + addr common.Address, start, end uint64, +) (uint64, error) { // if both block numbers are equal, return its value as birthblock if start == end { return start, nil @@ -99,7 +102,9 @@ func creationBlockInRange(client *Client, ctx context.Context, addr common.Addre // SourceCodeLenAt function returns the length of the current contract bytecode // at the block number provided. -func sourceCodeLenAt(client *Client, ctx context.Context, addr common.Address, atBlockNumber uint64) (int, error) { +func sourceCodeLenAt(client *web3.Client, ctx context.Context, + addr common.Address, atBlockNumber uint64, +) (int, error) { blockNumber := new(big.Int).SetUint64(atBlockNumber) sourceCode, err := client.CodeAt(ctx, addr, blockNumber) return len(sourceCode), err @@ -109,7 +114,7 @@ func sourceCodeLenAt(client *Client, ctx context.Context, addr common.Address, a // provided block numbers. It returns the logs, the last block scanned and an // error if any. It filters the logs by the topic hash and for the token // contract address provided. -func RangeOfLogs(ctx context.Context, client *Client, addr common.Address, +func RangeOfLogs(ctx context.Context, client *web3.Client, addr common.Address, fromBlock, lastBlock uint64, hexTopics ...string, ) ([]types.Log, uint64, bool, error) { // if the range is too big, scan only a part of it using the constant diff --git a/scanner/scanner.go b/scanner/scanner.go index 341d78bc..09937554 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -17,7 +17,8 @@ import ( "github.com/vocdoni/census3/db/annotations" queries "github.com/vocdoni/census3/db/sqlc" "github.com/vocdoni/census3/scanner/providers/manager" - "github.com/vocdoni/census3/scanner/providers/web3" + web3provider "github.com/vocdoni/census3/scanner/providers/web3" + "github.com/vocdoni/census3/helpers/web3" "go.vocdoni.io/dvote/log" ) @@ -315,7 +316,7 @@ func (s *Scanner) ScanHolders(ctx context.Context, token ScannerToken) ( qtx := s.db.QueriesRW.WithTx(tx) // if the provider is not an external one, instance the current token if !provider.IsExternal() { - if err := provider.SetRef(web3.Web3ProviderRef{ + if err := provider.SetRef(web3provider.Web3ProviderRef{ HexAddress: token.Address.Hex(), ChainID: token.ChainID, CreationBlock: token.CreationBlock,