diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74b9091810..8415c96558 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,6 +111,20 @@ jobs: file: packages/worker/Dockerfile no-cache: true + - name: Build and push Docker image for Data Fetcher + uses: docker/build-push-action@v4 + with: + push: true + tags: | + "matterlabs/block-explorer-data-fetcher:latest" + "matterlabs/block-explorer-data-fetcher:v${{ needs.createReleaseVersion.outputs.releaseVersion }}" + "matterlabs/block-explorer-data-fetcher:${{ steps.setVersionForFlux.outputs.imageTag }}" + "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:latest" + "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:v${{ needs.createReleaseVersion.outputs.releaseVersion }}" + "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:${{ steps.setVersionForFlux.outputs.imageTag }}" + file: packages/data-fetcher/Dockerfile + no-cache: true + - name: Build and push Docker image for App uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index 539a8e0679..a837a4a49e 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -66,6 +66,7 @@ jobs: packages/app/junit.xml packages/api/junit.xml packages/worker/junit.xml + packages/data-fetcher/junit.xml check_run_annotations: all tests, skipped tests report_individual_runs: "true" check_name: Unit Test Results diff --git a/.gitignore b/.gitignore index ecbf0116b8..50a0a84644 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ tests/e2e/reports/ # Logs logs !/packages/worker/test/logs/ +!/packages/data-fetcher/test/logs/ *.log npm-debug.log* yarn-debug.log* diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..55712c19f1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "typescript.tsdk": "node_modules/typescript/lib" +} \ No newline at end of file diff --git a/README.md b/README.md index e9276bde56..3cdd7e3dd5 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,9 @@

Online blockchain browser for viewing and analyzing zkSync Era blockchain.

## 📌 Overview -This repository is a monorepo consisting of 3 packages: -- [Worker](./packages/worker) - an indexer service for [zkSync Era](https://zksync.io) blockchain data. The purpose of the service is to read the data from the blockchain in real time, transform it and fill in it's database with the data in a way that makes it easy to be queried by the [API](./packages/api) service. +This repository is a monorepo consisting of 4 packages: +- [Worker](./packages/worker) - an indexer service for [zkSync Era](https://zksync.io) blockchain data. The purpose of the service is to read blockchain data in real time, transform it and fill in it's database with the data in a way that makes it easy to be queried by the [API](./packages/api) service. +- [Data Fetcher](./packages/data-fetcher) - a service that exposes and implements an HTTP endpoint to retrieve aggregated data for a certain block / range of blocks from the blockchain. This endpoint is called by the [Worker](./packages/worker) service. - [API](./packages/api) - a service providing Web API for retrieving structured [zkSync Era](https://zksync.io) blockchain data collected by [Worker](./packages/worker). It connects to the Worker's database to be able to query the collected data. - [App](./packages/app) - a front-end app providing an easy-to-use interface for users to view and inspect transactions, blocks, contracts and more. It makes requests to the [API](./packages/api) to get the data and presents it in a way that's easy to read and understand. @@ -20,10 +21,14 @@ flowchart subgraph explorer[Block explorer] Database[("Block explorer DB
(PostgreSQL)")] Worker(Worker service) + Data-Fetcher(Data Fetcher service) API(API service) App(App) - + + Worker-."Request aggregated data (HTTP)".->Data-Fetcher + Data-Fetcher-."Request data (HTTP)".->Blockchain Worker-.Save processed data.->Database + API-.Query data.->Database App-."Request data (HTTP)".->API App-."Request data (HTTP)".->Blockchain @@ -32,7 +37,7 @@ flowchart Worker-."Request data (HTTP)".->Blockchain ``` -[Worker](./packages/worker) service is responsible for getting data from blockchain using [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html), processing it and saving into the database. [API](./packages/api) service is connected to the same database where it gets the data from to handle API requests. It performs only read requests to the database. The front-end [App](./packages/app) makes HTTP calls to the Block Explorer [API](./packages/api) to get blockchain data and to the [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html) for reading contracts, performing transactions etc. +[Worker](./packages/worker) service retrieves aggregated data from the [Data Fetcher](./packages/data-fetcher) via HTTP and also directly from the blockchain using [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html), processes it and saves into the database. [API](./packages/api) service is connected to the same database where it gets the data from to handle API requests. It performs only read requests to the database. The front-end [App](./packages/app) makes HTTP calls to the Block Explorer [API](./packages/api) to get blockchain data and to the [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html) for reading contracts, performing transactions etc. ## 🚀 Features @@ -56,12 +61,12 @@ npm install ## ⚙️ Setting up env variables ### Manually set up env variables -Make sure you have set up all the necessary env variables. Follow [Setting up env variables for Worker](./packages/worker#setting-up-env-variables) and [Setting up env variables for API](./packages/api#setting-up-env-variables) for instructions. For the [App](./packages/app) package you might want to edit environment config, see [Environment configs](./packages/app#environment-configs). +Make sure you have set up all the necessary env variables. Follow setting up env variables instructions for [Worker](./packages/worker#setting-up-env-variables), [Data Fetcher](./packages/data-fetcher#setting-up-env-variables) and [API](./packages/api#setting-up-env-variables). For the [App](./packages/app) package you might want to edit environment config, see [Environment configs](./packages/app#environment-configs). ### Build env variables based on your [zksync-era](https://github.com/matter-labs/zksync-era) local repo setup Make sure you have [zksync-era](https://github.com/matter-labs/zksync-era) repo set up locally. You must have your environment variables files present in the [zksync-era](https://github.com/matter-labs/zksync-era) repo at `/etc/env/*.env` for the build envs script to work. -The following script sets `.env` files for [Worker](./packages/worker) and [API](./packages/api) packages as well as environment configuration file for [App](./packages/app) package based on your local [zksync-era](https://github.com/matter-labs/zksync-era) repo setup. +The following script sets `.env` files for [Worker](./packages/worker), [Data Fetcher](./packages/data-fetcher) and [API](./packages/api) packages as well as environment configuration file for [App](./packages/app) package based on your local [zksync-era](https://github.com/matter-labs/zksync-era) repo setup. ```bash npm run hyperchain:configure ``` @@ -75,7 +80,7 @@ To create a database run the following command: npm run db:create ``` -To run all the packages (`Worker`, `API` and front-end `App`) in `development` mode run the following command from the root directory. +To run all the packages (`Worker`, `Data Fetcher`, `API` and front-end `App`) in `development` mode run the following command from the root directory. ```bash npm run dev ``` @@ -100,7 +105,7 @@ To get block-explorer connected to your ZK Stack Hyperchain you need to set up a ## 🔍 Verify Block Explorer is up and running -To verify front-end `App` is running open http://localhost:3010 in your browser. `API` should be available at http://localhost:3020. `Worker` - http://localhost:3001. +To verify front-end `App` is running open http://localhost:3010 in your browser. `API` should be available at http://localhost:3020, `Worker` at http://localhost:3001 and `Data Fetcher` at http://localhost:3040. ## 🕵️‍♂️ Testing Run unit tests for all packages: diff --git a/docker-compose-cli.yaml b/docker-compose-cli.yaml index 1343eb8fa2..c01ce5d059 100644 --- a/docker-compose-cli.yaml +++ b/docker-compose-cli.yaml @@ -1,10 +1,10 @@ -version: '3.2' +version: "3.2" +name: "zkcli-block-explorer" services: app: - build: - context: . - dockerfile: ./packages/app/Dockerfile + platform: linux/amd64 + image: "matterlabs/block-explorer-app:${VERSION}" ports: - '3010:3010' depends_on: @@ -12,9 +12,8 @@ services: restart: unless-stopped worker: - build: - context: . - dockerfile: ./packages/worker/Dockerfile + platform: linux/amd64 + image: "matterlabs/block-explorer-worker:${VERSION}" environment: - PORT=3001 - LOG_LEVEL=verbose @@ -23,14 +22,26 @@ services: - DATABASE_USER=postgres - DATABASE_PASSWORD=postgres - DATABASE_NAME=block-explorer - - BLOCKCHAIN_RPC_URL=http://host.docker.internal:3050 + - BLOCKCHAIN_RPC_URL=http://host.docker.internal:${RPC_PORT} + - DATA_FETCHER_URL=http://data-fetcher:3040 - BATCHES_PROCESSING_POLLING_INTERVAL=1000 restart: unless-stopped + data-fetcher: + platform: linux/amd64 + image: "matterlabs/block-explorer-data-fetcher:${VERSION}" + environment: + - PORT=3040 + - LOG_LEVEL=verbose + - NODE_ENV=development + - BLOCKCHAIN_RPC_URL=http://host.docker.internal:${RPC_PORT} + ports: + - '3040:3040' + restart: unless-stopped + api: - build: - context: . - dockerfile: ./packages/api/Dockerfile + platform: linux/amd64 + image: "matterlabs/block-explorer-api:${VERSION}" environment: - PORT=3020 - METRICS_PORT=3005 @@ -60,4 +71,4 @@ services: - POSTGRES_DB=block-explorer volumes: - postgres: + postgres: \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index cd747b2e49..1020cc35bc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,6 +30,7 @@ services: - DATABASE_PASSWORD=postgres - DATABASE_NAME=block-explorer - BLOCKCHAIN_RPC_URL=http://zksync:3050 + - DATA_FETCHER_URL=http://data-fetcher:3040 - BATCHES_PROCESSING_POLLING_INTERVAL=1000 ports: - '3001:3001' @@ -43,6 +44,29 @@ services: condition: service_healthy restart: unless-stopped + data-fetcher: + build: + context: . + dockerfile: ./packages/data-fetcher/Dockerfile + target: development-stage + command: npm run --prefix packages/data-fetcher dev:debug + environment: + - PORT=3040 + - LOG_LEVEL=verbose + - NODE_ENV=development + - BLOCKCHAIN_RPC_URL=http://zksync:3050 + ports: + - '3040:3040' + - '9231:9229' + - '9232:9230' + volumes: + - ./packages/data-fetcher:/usr/src/app/packages/data-fetcher + - /usr/src/app/packages/data-fetcher/node_modules + depends_on: + zksync: + condition: service_healthy + restart: unless-stopped + api: build: context: . @@ -58,8 +82,8 @@ services: ports: - '3020:3020' - '3005:3005' - - '9231:9229' - - '9232:9230' + - '9233:9229' + - '9234:9230' volumes: - ./packages/api:/usr/src/app/packages/api - /usr/src/app/packages/api/node_modules diff --git a/package-lock.json b/package-lock.json index b5e9876c27..28760ca985 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23466,6 +23466,10 @@ "node": ">=8" } }, + "node_modules/data-fetcher": { + "resolved": "packages/data-fetcher", + "link": true + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -55159,6 +55163,205 @@ "typedarray-to-buffer": "^3.1.5" } }, + "packages/data-fetcher": { + "version": "0.0.0", + "license": "MIT", + "dependencies": { + "@nestjs/common": "^9.0.0", + "@nestjs/config": "^2.2.0", + "@nestjs/core": "^9.0.0", + "@nestjs/platform-express": "^9.0.0", + "@nestjs/terminus": "^9.1.2", + "@willsoto/nestjs-prometheus": "^4.7.0", + "ethers": "^5.7.1", + "nest-winston": "^1.7.0", + "prom-client": "^14.1.0", + "reflect-metadata": "^0.1.13", + "rimraf": "^3.0.2", + "rxjs": "^7.2.0", + "winston": "^3.8.2", + "zksync-web3": "0.15.4" + }, + "devDependencies": { + "@nestjs/cli": "^9.0.0", + "@nestjs/schematics": "^9.0.0", + "@nestjs/testing": "^9.0.0", + "@types/express": "^4.17.13", + "@types/jest": "28.1.8", + "@types/supertest": "^2.0.11", + "@typescript-eslint/eslint-plugin": "^5.0.0", + "@typescript-eslint/parser": "^5.0.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-prettier": "^4.0.0", + "jest": "29.2.1", + "jest-junit": "^14.0.1", + "jest-mock-extended": "^3.0.1", + "lint-staged": "^13.0.3", + "source-map-support": "^0.5.20", + "supertest": "^6.1.3", + "ts-jest": "29.0.3", + "ts-loader": "^9.2.3", + "ts-node": "^10.0.0", + "tsconfig-paths": "4.1.0" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=9.0.0" + } + }, + "packages/data-fetcher/node_modules/@ethersproject/networks": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/@ethersproject/networks/-/networks-5.7.1.tgz", + "integrity": "sha512-n/MufjFYv3yFcUyfhnXotyDlNdFb7onmkSy8aQERi2PjNcnWQ66xXxa3XlS8nCcA8aJKJjIIMNJTC7tu80GwpQ==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/logger": "^5.7.0" + } + }, + "packages/data-fetcher/node_modules/@ethersproject/providers": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/@ethersproject/providers/-/providers-5.7.2.tgz", + "integrity": "sha512-g34EWZ1WWAVgr4aptGlVBF8mhl3VWjv+8hoAnzStu8Ah22VHBsuGzP17eb6xDVRzw895G4W7vvx60lFFur/1Rg==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/abstract-provider": "^5.7.0", + "@ethersproject/abstract-signer": "^5.7.0", + "@ethersproject/address": "^5.7.0", + "@ethersproject/base64": "^5.7.0", + "@ethersproject/basex": "^5.7.0", + "@ethersproject/bignumber": "^5.7.0", + "@ethersproject/bytes": "^5.7.0", + "@ethersproject/constants": "^5.7.0", + "@ethersproject/hash": "^5.7.0", + "@ethersproject/logger": "^5.7.0", + "@ethersproject/networks": "^5.7.0", + "@ethersproject/properties": "^5.7.0", + "@ethersproject/random": "^5.7.0", + "@ethersproject/rlp": "^5.7.0", + "@ethersproject/sha2": "^5.7.0", + "@ethersproject/strings": "^5.7.0", + "@ethersproject/transactions": "^5.7.0", + "@ethersproject/web": "^5.7.0", + "bech32": "1.1.4", + "ws": "7.4.6" + } + }, + "packages/data-fetcher/node_modules/@ethersproject/web": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/@ethersproject/web/-/web-5.7.1.tgz", + "integrity": "sha512-Gueu8lSvyjBWL4cYsWsjh6MtMwM0+H4HvqFPZfB6dV8ctbP9zFAO73VG1cMWae0FLPCtz0peKPpZY8/ugJJX2w==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/base64": "^5.7.0", + "@ethersproject/bytes": "^5.7.0", + "@ethersproject/logger": "^5.7.0", + "@ethersproject/properties": "^5.7.0", + "@ethersproject/strings": "^5.7.0" + } + }, + "packages/data-fetcher/node_modules/ethers": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/ethers/-/ethers-5.7.2.tgz", + "integrity": "sha512-wswUsmWo1aOK8rR7DIKiWSw9DbLWe6x98Jrn8wcTflTVvaXhAMaB5zGAXy0GYQEQp9iO1iSHWVyARQm11zUtyg==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/abi": "5.7.0", + "@ethersproject/abstract-provider": "5.7.0", + "@ethersproject/abstract-signer": "5.7.0", + "@ethersproject/address": "5.7.0", + "@ethersproject/base64": "5.7.0", + "@ethersproject/basex": "5.7.0", + "@ethersproject/bignumber": "5.7.0", + "@ethersproject/bytes": "5.7.0", + "@ethersproject/constants": "5.7.0", + "@ethersproject/contracts": "5.7.0", + "@ethersproject/hash": "5.7.0", + "@ethersproject/hdnode": "5.7.0", + "@ethersproject/json-wallets": "5.7.0", + "@ethersproject/keccak256": "5.7.0", + "@ethersproject/logger": "5.7.0", + "@ethersproject/networks": "5.7.1", + "@ethersproject/pbkdf2": "5.7.0", + "@ethersproject/properties": "5.7.0", + "@ethersproject/providers": "5.7.2", + "@ethersproject/random": "5.7.0", + "@ethersproject/rlp": "5.7.0", + "@ethersproject/sha2": "5.7.0", + "@ethersproject/signing-key": "5.7.0", + "@ethersproject/solidity": "5.7.0", + "@ethersproject/strings": "5.7.0", + "@ethersproject/transactions": "5.7.0", + "@ethersproject/units": "5.7.0", + "@ethersproject/wallet": "5.7.0", + "@ethersproject/web": "5.7.1", + "@ethersproject/wordlists": "5.7.0" + } + }, + "packages/data-fetcher/node_modules/ws": { + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "packages/data-fetcher/node_modules/zksync-web3": { + "version": "0.15.4", + "resolved": "https://registry.npmjs.org/zksync-web3/-/zksync-web3-0.15.4.tgz", + "integrity": "sha512-6CEpRBbF4nGwRYSF3KvPGqg2aNJFYTl8AR+cejBnC2Uyu1v3NYSkmkXXVuMGupJ7HIQR1aTqFEDsUFPyO/bL0Q==", + "deprecated": "This package has been deprecated in favor of zksync-ethers@5.0.0", + "peerDependencies": { + "ethers": "^5.7.0" + } + }, "packages/worker": { "version": "0.0.0", "license": "MIT", diff --git a/packages/app/src/components/NetworkDeprecated.vue b/packages/app/src/components/NetworkDeprecated.vue index 8b9e8a7ea3..de32dc159d 100644 --- a/packages/app/src/components/NetworkDeprecated.vue +++ b/packages/app/src/components/NetworkDeprecated.vue @@ -11,7 +11,6 @@