move web3 endpoints poll from scanner/providers/web3 package to helpe…

…rs package

api/api.go

@@ -17,8 +17,9 @@ import (
 	"github.com/vocdoni/census3/db/annotations"
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"github.com/vocdoni/census3/helpers/queue"
+	"github.com/vocdoni/census3/helpers/web3"
 	"github.com/vocdoni/census3/scanner/providers"
-	"github.com/vocdoni/census3/scanner/providers/web3"
+	web3provider "github.com/vocdoni/census3/scanner/providers/web3"
 	"go.vocdoni.io/dvote/api/censusdb"
 	storagelayer "go.vocdoni.io/dvote/data"
 	"go.vocdoni.io/dvote/data/downloader"
@@ -247,7 +248,7 @@ func (capi *census3API) CreateInitialTokens(tokensPath string) error {
 			continue
 		}
 		if !provider.IsExternal() {
-			if err := provider.SetRef(web3.Web3ProviderRef{
+			if err := provider.SetRef(web3provider.Web3ProviderRef{
 				HexAddress: token.ID,
 				ChainID:    token.ChainID,
 			}); err != nil {

cmd/census3/main.go

@@ -22,7 +22,8 @@ import (
 	gitcoinDB "github.com/vocdoni/census3/scanner/providers/gitcoin/db"
 	"github.com/vocdoni/census3/scanner/providers/manager"
 	"github.com/vocdoni/census3/scanner/providers/poap"
-	"github.com/vocdoni/census3/scanner/providers/web3"
+	web3provider "github.com/vocdoni/census3/scanner/providers/web3"
+	"github.com/vocdoni/census3/helpers/web3"
 	"go.vocdoni.io/dvote/log"
 )
 
@@ -158,10 +159,10 @@ func main() {
 	// init the provider manager
 	pm := manager.NewProviderManager()
 	// init the web3 token providers
-	web3ProviderConf := web3.Web3ProviderConfig{Endpoints: w3p}
-	pm.AddProvider(new(web3.ERC20HolderProvider).Type(), web3ProviderConf)
-	pm.AddProvider(new(web3.ERC721HolderProvider).Type(), web3ProviderConf)
-	pm.AddProvider(new(web3.ERC777HolderProvider).Type(), web3ProviderConf)
+	web3ProviderConf := web3provider.Web3ProviderConfig{Endpoints: w3p}
+	pm.AddProvider(new(web3provider.ERC20HolderProvider).Type(), web3ProviderConf)
+	pm.AddProvider(new(web3provider.ERC721HolderProvider).Type(), web3ProviderConf)
+	pm.AddProvider(new(web3provider.ERC777HolderProvider).Type(), web3ProviderConf)
 	// init POAP external provider
 	if config.poapAPIEndpoint != "" {
 		pm.AddProvider(new(poap.POAPHolderProvider).Type(), poap.POAPConfig{

scanner/providers/web3/web3_pool.go → helpers/web3/web3_pool.go

@@ -18,10 +18,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/ethereum/go-ethereum/ethclient"
 )
 
+const (
+	DefaultMaxWeb3ClientRetries = 5
+
+	shortNameSourceUri        = "https://chainid.network/chains_mini.json"
+	checkWeb3EndpointsTimeout = time.Second * 10
+)
+
 // Web3Pool struct contains a map of chainID-[]*Web3Endpoint, where
 // the key is the chainID and the value is a list of Web3Endpoint. It also
 // contains a list of all the Web3Endpoint metadata. It provides methods to

scanner/providers/farcaster/types.go

@@ -9,7 +9,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	fcir "github.com/vocdoni/census3/contracts/farcaster/idRegistry"
 	fckr "github.com/vocdoni/census3/contracts/farcaster/keyRegistry"
-	"github.com/vocdoni/census3/scanner/providers/web3"
+	"github.com/vocdoni/census3/helpers/web3"
 )
 
 type FarcasterProviderConf struct {

scanner/providers/web3/const.go

@@ -1,19 +1,17 @@
 package web3
 
-import "time"
+import (
+	"time"
 
-const (
-	DefaultMaxWeb3ClientRetries = 5
-	RetryWeb3Cooldown           = 500 * time.Millisecond
+	"github.com/vocdoni/census3/helpers/web3"
 )
 
 const (
-	shortNameSourceUri        = "https://chainid.network/chains_mini.json"
-	checkWeb3EndpointsTimeout = time.Second * 10
-	TimeLayout                = "2006-01-02T15:04:05Z07:00"
+	RetryWeb3Cooldown = 500 * time.Millisecond
+	TimeLayout        = "2006-01-02T15:04:05Z07:00"
 )
 
-var DefaultWeb3Endpoint = &Web3Endpoint{
+var DefaultWeb3Endpoint = &web3.Web3Endpoint{
 	ChainID:   11155111,
 	Name:      "Sepolia",
 	ShortName: "sep",

scanner/providers/web3/erc20_provider.go

@@ -11,12 +11,13 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	erc20 "github.com/vocdoni/census3/contracts/erc/erc20"
 	"github.com/vocdoni/census3/scanner/providers"
+	"github.com/vocdoni/census3/helpers/web3"
 	"go.vocdoni.io/dvote/log"
 )
 
 type ERC20HolderProvider struct {
-	endpoints *Web3Pool
-	client    *Client
+	endpoints *web3.Web3Pool
+	client    *web3.Client
 
 	contract         *erc20.ERC20Contract
 	address          common.Address

scanner/providers/web3/erc721_provider.go

@@ -11,12 +11,13 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	erc721 "github.com/vocdoni/census3/contracts/erc/erc721"
 	"github.com/vocdoni/census3/scanner/providers"
+	"github.com/vocdoni/census3/helpers/web3"
 	"go.vocdoni.io/dvote/log"
 )
 
 type ERC721HolderProvider struct {
-	endpoints *Web3Pool
-	client    *Client
+	endpoints *web3.Web3Pool
+	client    *web3.Client
 
 	contract         *erc721.ERC721Contract
 	address          common.Address

scanner/providers/web3/erc777_provider.go

@@ -11,12 +11,13 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	erc777 "github.com/vocdoni/census3/contracts/erc/erc777"
 	"github.com/vocdoni/census3/scanner/providers"
+	"github.com/vocdoni/census3/helpers/web3"
 	"go.vocdoni.io/dvote/log"
 )
 
 type ERC777HolderProvider struct {
-	endpoints *Web3Pool
-	client    *Client
+	endpoints *web3.Web3Pool
+	client    *web3.Client
 
 	contract         *erc777.ERC777Contract
 	address          common.Address

scanner/providers/web3/web3_provider.go

@@ -12,6 +12,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vocdoni/census3/scanner/providers"
+	"github.com/vocdoni/census3/helpers/web3"
 	"go.vocdoni.io/dvote/db"
 	"go.vocdoni.io/dvote/log"
 )
@@ -24,19 +25,19 @@ type Web3ProviderRef struct {
 
 type Web3ProviderConfig struct {
 	Web3ProviderRef
-	Endpoints *Web3Pool
+	Endpoints *web3.Web3Pool
 	DB        *db.Database
 }
 
 // creationBlock function returns the block number of the creation of a contract
 // address. It uses the `eth_getCode` method to get the contract code at the
 // block number provided. If the method is not supported, it returns 0 and nil.
-func creationBlock(client *Client, ctx context.Context, addr common.Address) (uint64, error) {
+func creationBlock(client *web3.Client, ctx context.Context, addr common.Address) (uint64, error) {
 	// check if the current client supports `eth_getCode` method, if not, return
 	// 1 and nil. It is assumed that the contract is created at block 1 to start
 	// scanning from the first block.
 	getCodeSupport := false
-	for i := 0; i < DefaultMaxWeb3ClientRetries; i++ {
+	for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ {
 		ethClient, err := client.EthClient()
 		if err != nil {
 			return 0, err
@@ -52,7 +53,7 @@ func creationBlock(client *Client, ctx context.Context, addr common.Address) (ui
 	// get the latest block number
 	var err error
 	var lastBlock uint64
-	for i := 0; i < DefaultMaxWeb3ClientRetries; i++ {
+	for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ {
 		lastBlock, err = client.BlockNumber(ctx)
 		if err == nil {
 			break
@@ -63,7 +64,7 @@ func creationBlock(client *Client, ctx context.Context, addr common.Address) (ui
 		return 0, err
 	}
 	var creationBlock uint64
-	for i := 0; i < DefaultMaxWeb3ClientRetries; i++ {
+	for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ {
 		creationBlock, err = creationBlockInRange(client, ctx, addr, 0, lastBlock)
 		if err == nil {
 			break
@@ -75,7 +76,9 @@ func creationBlock(client *Client, ctx context.Context, addr common.Address) (ui
 
 // creationBlockInRange function finds the block number of a contract between
 // the bounds provided as start and end blocks.
-func creationBlockInRange(client *Client, ctx context.Context, addr common.Address, start, end uint64) (uint64, error) {
+func creationBlockInRange(client *web3.Client, ctx context.Context,
+	addr common.Address, start, end uint64,
+) (uint64, error) {
 	// if both block numbers are equal, return its value as birthblock
 	if start == end {
 		return start, nil
@@ -99,7 +102,9 @@ func creationBlockInRange(client *Client, ctx context.Context, addr common.Addre
 
 // SourceCodeLenAt function returns the length of the current contract bytecode
 // at the block number provided.
-func sourceCodeLenAt(client *Client, ctx context.Context, addr common.Address, atBlockNumber uint64) (int, error) {
+func sourceCodeLenAt(client *web3.Client, ctx context.Context,
+	addr common.Address, atBlockNumber uint64,
+) (int, error) {
 	blockNumber := new(big.Int).SetUint64(atBlockNumber)
 	sourceCode, err := client.CodeAt(ctx, addr, blockNumber)
 	return len(sourceCode), err
@@ -109,7 +114,7 @@ func sourceCodeLenAt(client *Client, ctx context.Context, addr common.Address, a
 // provided block numbers. It returns the logs, the last block scanned and an
 // error if any. It filters the logs by the topic hash and for the token
 // contract address provided.
-func RangeOfLogs(ctx context.Context, client *Client, addr common.Address,
+func RangeOfLogs(ctx context.Context, client *web3.Client, addr common.Address,
 	fromBlock, lastBlock uint64, hexTopics ...string,
 ) ([]types.Log, uint64, bool, error) {
 	// if the range is too big, scan only a part of it using the constant

scanner/scanner.go

@@ -17,7 +17,8 @@ import (
 	"github.com/vocdoni/census3/db/annotations"
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"github.com/vocdoni/census3/scanner/providers/manager"
-	"github.com/vocdoni/census3/scanner/providers/web3"
+	web3provider "github.com/vocdoni/census3/scanner/providers/web3"
+	"github.com/vocdoni/census3/helpers/web3"
 	"go.vocdoni.io/dvote/log"
 )
 
@@ -315,7 +316,7 @@ func (s *Scanner) ScanHolders(ctx context.Context, token ScannerToken) (
 	qtx := s.db.QueriesRW.WithTx(tx)
 	// if the provider is not an external one, instance the current token
 	if !provider.IsExternal() {
-		if err := provider.SetRef(web3.Web3ProviderRef{
+		if err := provider.SetRef(web3provider.Web3ProviderRef{
 			HexAddress:    token.Address.Hex(),
 			ChainID:       token.ChainID,
 			CreationBlock: token.CreationBlock,