Merge pull request #26 from covalenthq/fixes/pre-release-1.0.0

[BSP Agent]Fixes pre release 1.0.0
covalenthq · Jan 26, 2022 · a399bff · a399bff
2 parents 770b07f + d249e1f
commit a399bff
Show file tree

Hide file tree

Showing 12 changed files with 329 additions and 75 deletions.
diff --git a/Makefile b/Makefile
@@ -1,11 +1,13 @@
 docker:
 	@docker-compose down
-	@docker-compose build
-	@docker-compose up -d
+	@docker-compose -f "docker-compose.yml" up --build --remove-orphans --force-recreate --exit-code-from consumer
 
 dockerdown:
 	@docker-compose down
 
+lint:
+	@golangci-lint run
+
 run-build:
 	@echo "---- Building Agent from cmd/mqstoreagent ----"
 	@go build -o ./bin/mqstoreagent/agent ./cmd/mqstoreagent/*.go 
@@ -15,11 +17,11 @@ run-agent-eth:
 	@echo "---- Running Agent from cmd/mqstoreagent ----"
 	@go run ./cmd/mqstoreagent/*.go \
 	--redis-url="redis://username:@localhost:6379/0?topic=replication#replicate" \
-	--codec-path="./codec/block-ethereum.avsc" \
+	--avro-codec-path="./codec/block-ethereum.avsc" \
 	--binary-file-path="./bin/block-ethereum/" \
 	--gcp-svc-account="/Users/pranay/.config/gcloud/bsp-2.json" \
 	--replica-bucket="covalenthq-geth-block-specimen" \
-	--segment-length=5 \
+	--segment-length=1 \
  	--eth-client=http://127.0.0.1:7545  \
  	--proof-chain-address=0x3D25EBCeFC7F1E5a5664C8D6AA63Dc3D513761D6 \
 	--consumer-timeout=80
@@ -28,12 +30,12 @@ run-agent-elrond:
 	@echo "---- Running Agent from cmd/mqstoreagent ----"
 	@go run ./cmd/mqstoreagent/*.go \
 	--redis-url="redis://username:@localhost:6379/0?topic=replication#replicate" \
-   	--codec-path="./codec/block-elrond.avsc" \
-    --binary-file-path="./bin/block-elrond/" \
-    --gcp-svc-account="/Users/pranay/.config/gcloud/bsp.json" \
-    --replica-bucket="covalenthq-geth-block-specimen" \
-    --segment-length=5 \
- 	--eth-client=http://127.0.0.1:7545  \
- 	--proof-chain-address=0x3D25EBCeFC7F1E5a5664C8D6AA63Dc3D513761D6 \
+	--avro-codec-path="./codec/block-elrond.avsc" \
+	--binary-file-path="./bin/block-elrond/" \
+	--gcp-svc-account="/Users/pranay/.config/gcloud/bsp.json" \
+	--replica-bucket="covalenthq-geth-block-specimen" \
+	--segment-length=1 \
+	--eth-client=http://127.0.0.1:7545  \
+	--proof-chain-address=0x3D25EBCeFC7F1E5a5664C8D6AA63Dc3D513761D6 \
 	--consumer-timeout=80 \
 	--websocket-urls="34.66.210.112:20000 34.66.210.112:20001 34.66.210.112:20002 34.66.210.112:20003" 
diff --git a/README.md b/README.md
@@ -1,28 +1,83 @@
-# BSP-agent (mq-store-agent)
+![banner](./docs/covalent.jpg)
 
-Reads block-replicas (block-results, block-specimens, or any other types) produced by geth block specimen/result producers streamed into a redis stream, decodes them from RLP encoding, creates a proof transaction on a virtual chain with a sha-256 checksum of the data in the object, and persists them into object storage for a specified google bucket and saves it to a file - atomically.
+<div align="center">
+  <a href="https://github.com/covalenthq/mq-store-agent/releases/latest">
+    <img alt="Version" src="https://img.shields.io/badge/tag-v1.0.0-yellowgreen" />
+  </a>
+  <a href="https://github.com/covalenthq/mq-store-agent/blob/main/LICENSE">
+    <img alt="License: " src="https://img.shields.io/badge/license-MIT-green" />
+  </a>
+  <a href="https://goreportcard.com/report/github.com/covalenthq/mq-store-agent">
+    <img alt="Go report card" src="https://goreportcard.com/badge/github.com/covalenthq/mq-store-agent"/>
+  </a>
+  <!-- <a href="https://bestpractices.coreinfrastructure.org/projects/0000">
+    <img alt="Lines of code" src="https://img.shields.io/tokei/lines/github/covalenthq/mq-store-agent">
+  </a> -->
+  <a href="http://covalenthq.com/discord">
+    <img alt="Discord" src="https://img.shields.io/badge/discord-join%20chat-blue.svg" />
+  </a>
+</div>
+<div align="center">
+  <a href="http://covalenthq.com/discord">
+    <img alt="Discord" src="https://img.shields.io/discord/715804406842392586.svg" />
+  </a>
+  <a href="https://github.com/covalenthq/mq-store-agent/actions/workflows/golangci-lint.yml?query=branch%3Amain+workflow%3Agolangci-lint">
+    <img alt="Lint Status" src="https://github.com/covalenthq/mq-store-agent/actions/workflows/golangci-lint.yml/badge.svg?branch=main" />
+  </a>
+  <a href="https://github.com/covalenthq/mq-store-agent/actions/workflows/docker-image.yml?query=branch%3Amain+workflow%3Adocker-image-ci">
+    <img alt="CI Status" src="https://github.com/covalenthq/mq-store-agent/actions/workflows/docker-image.yml/badge.svg?branch=main" />
+  </a>
+  <a href="https://twitter.com/@Covalent_HQ">
+    <img alt="Twitter Follow Covalent" src="https://img.shields.io/twitter/follow/Covalent_HQ"/>
+  </a>
+</div>
 
-Please refer to these internal [instructions](https://docs.google.com/document/d/1BMC9-VXZfpB6mGczSu8ylUXJZ_CIx4ephepDtlruv_Q/edit?usp=sharing) for running the BSP with the mq-store-agent (BSP-agent).
+# BSP Agent (mq-store-agent)
 
-Please refer to this internal [whitepaper](https://docs.google.com/document/d/1J6RalVVfMSh2kSKNHM3Agb4GngzWVw9e1PqLSVb3-PU/edit#) to understand more about its function.
+* [Introduction](#agent_intro)
+  * [Resources](#agent_resources)
+* [Architecture](#agent_arch)
+* [Block-Replica](#agent_block)
+  * [State-Specimen](#state_specimen)
+* [Environment](#environment)
+* [Build & Run](#build_run)
+  * [Flag Definitions](#flag_definitions)
+* [Docker](#docker)
+* [Scripts](#scripts)
+  * [Inspect](#inspect)
+* [Contributing](./docs/CONTRIBUTING.md)
 
-Externally facing validator [documentation.](https://www.notion.so/covalenthq/Validator-Documentation-e9fdba94c9e149aeba798ece303dc5d4)
+## <span id="agent_intro">Introduction</span>
 
-Externally facing workshop [deck.](https://docs.google.com/presentation/d/1qInReJcMxvVywJ8onoFPoKCwuorJ8LpOn3hwLJIl7bg/edit?usp=sharing)
+Decodes, packs, encodes, proves, stores and uploads block-replicas (can be block-results, block-specimens, or any other pre-defined block types), which are primarily "block-specimens" produced by EVM or non-EVM byte code based blockchains.
 
-## Architecture
+These block-replicas are produced by go-ethereum nodes / websocket block data sources modified with block-specimen producers(BSP) streamed into a redis channel. The agent first decodes them from their native RLP encoding, repacks them into segments of bigger chunks containing more than one block's worth of data, creates a proof transaction on the proof-chain smart contract (also called cqt-virtnet) with a sha-256 checksum of the data contained in the object, and finally persists them into object storage (local and cloud) for a specified google bucket - atomically.
 
-![diagram](arch.png)
+## <span id="agent_resources">Resources</span>
 
-## Block-replica
+Production of Block Specimen forms the core of the network’s data objects specification. These objects are created with the aid of three main pieces of open-source software provided by Covalent for the network’s decentralized stack.
 
-Block Replicas are created by [BSP](https://docs.google.com/document/d/1BMC9-VXZfpB6mGczSu8ylUXJZ_CIx4ephepDtlruv_Q/edit#heading=h.5owqpz3w99gp) here and fed into [Redis streams](https://redis.io/topics/streams-intro).
+1. [Block Specimen Producer (BSP Geth)](https://github.com/covalenthq/go-ethereum) - Operator run & deployed
 
-These objects are extracted and read into the following struct by the agent. There are three types of objects currently -
+1. [BSP Agent](https://github.com/covalenthq/mq-store-agent) - Operator run & deployed
 
-1. block-replica - have all the data shown below.
-1. block-result - have all the data shown below except "State".
-1. block-specimen - have all the data shown below except "TotalDifficulty", "Receipts"(&"Logs"), "Senders"
+1. [BSP Proof-chain](https://github.com/covalenthq/cqt-virtnet) - Covalent operated & pre-deployed
+
+Please refer to these [instructions](https://docs.google.com/document/d/1N_HxUi6ZEkub9EHANe49vkL9iQztVA_ACyfHcOZV5y0/edit?usp=sharing) for running the BSP with the mq-store-agent (BSP Agent).
+
+Please refer to this [whitepaper](https://www.covalenthq.com/static/documents/Block%20Specimen%20Whitepaper%20V1.1.pdf) to understand more about its function.
+
+## <span id="agent_arch">Architecture</span>
+
+![diagram](./docs/arch.png)
+
+## <span id="agent_block">Block-replica</span>
+
+Block Replicas are created by the [BSP](https://docs.google.com/document/d/1BMC9-VXZfpB6mGczSu8ylUXJZ_CIx4ephepDtlruv_Q/edit#heading=h.5owqpz3w99gp) here and fed into [Redis streams](https://redis.io/topics/streams-intro).
+
+These objects are extracted and read into the following struct by the agent. There are two types of block-replica objects currently -
+
+For Ethereum -
 
 ```go
     type BlockReplica struct {
@@ -39,10 +94,25 @@ These objects are extracted and read into the following struct by the agent. The
     }
 ```
 
-### State-specimen
+For Elrond -
+
+```go
+    type ElrondBlockReplica struct {
+        Block        *Block
+        Transactions []*ElrondTransaction
+        SCResults    []*SCResult
+        Receipts     []*ElrondReceipt
+        Logs         []*Log
+        StateChanges []*AccountBalanceUpdate
+    }
+```
+
+### <span id="state_specimen">State-specimen</span>
 
 The "State" is comprised of all state information related to accounts ever touched for a given block.
 
+For Ethereum -
+
 ```go
     type StateSpecimen struct {
         AccountRead []*accountRead
@@ -66,9 +136,19 @@ The "State" is comprised of all state information related to accounts ever touch
     }
 ```
 
-## Environment
+For Elrond -
+
+```go
+    type AccountBalanceUpdate struct {
+        Address []byte
+        Balance []byte
+        Nonce   int64
+    }
+```
 
-The Eth private key allows the validator to make proof transactions to the proof-chain contract and is required. Other env vars are optional depending on your redis, eth account configuration. Add the following to your `.envrc` at the root dir with final relative path `~/mq-store-agent/envrc`
+## <span id="environment">Environment</span>
+
+An Ethereum private key (for a public address that is pre-whitelisted on the staking contract) allows block-specimen producers (operators) to make proof transactions to the proof-chain contract and is required by the mq-store-agent. Other env vars are optional depending on your redis, eth account configuration. Add the following to your `.envrc` at the root dir with final relative path `~/mq-store-agent/.envrc`
 
 ```bash
     export ETH_PRIVATE_KEY=private/key/senders #required
@@ -92,41 +172,72 @@ For which you should see something like -
     direnv: export +ETH_PRIVATE_KEY
 ```
 
-## Build & Run
+The remaining environment configuration is set up with flags provided to the mq-store-agent during runtime.
+
+## <span id="build_run">Build & Run</span>
 
-Run the agent directly with the following -
+Clone the `covalenthq/mq-store-agent` repo and checkout `main`
 
 ```bash
-go run ./cmd/mqstoreagent/*.go \ 
-    --redis-url="redis://username:@localhost:6379/0?topic=replication#replicate" \ 
-    --codec-path="./codec/block-replica.avsc" \ 
-    --binary-file-path="./bin/block-replica/" \ 
-    --gcp-svc-account="/Users/<user>/.config/gcloud/<gcp-service-account.json>" \ 
-    --replica-bucket="<covalenthq-geth-block-replica-bucket>" \ 
-    --segment-length=5 \ 
-    --eth-client="http://127.0.0.1:7545" \ 
-    --proof-chain-address="0xb5B12cbe8bABAF96677F60f65317b81709062C47"
+git clone git@github.com:covalenthq/mq-store-agent.git
+cd mq-store-agent
+git checkout main
+```
+
+Run the agent for (ethereum block-specimens) locally directly using the following -
+
+```bash
+go run ./cmd/mqstoreagent/*.go \
+    --redis-url="redis://username:@localhost:6379/0?topic=replication#replicate" \
+    --avro-codec-path="./codec/block-ethereum.avsc" \
+    --binary-file-path="./bin/block-ethereum/" \
+    --gcp-svc-account="/Users/<user>/.config/gcloud/<gcp-service-account.json>" \
+    --replica-bucket="<covalenthq-geth-block-replica-bucket>" \
+    --segment-length=1 \
+    --eth-client="http://127.0.0.1:7545" \
+    --proof-chain-address="0xb5B12cbe8bABAF96677F60f65317b81709062C47" \
+    --consumer-timeout=80
 ```
 
 Or update the Makefile with the correct --gcp-svc-account, --replica-bucket & --proof-chain-address and run with the following.
 
 ```bash
-    make run-build
-    make run-agent
+    make run-agent-eth
 ```
 
-### Docker
+### <span id="flag_definitions">Flag definitions</span>
 
-The docker image for this service can be found [here](https://github.com/covalenthq/mq-store-agent/pkgs/container/mq-store-agent)
+--redis-url - this flag tells the BSP agent where to find the BSP messages, the stream topic key `replication` and the consumer group name with the field after "#" that in this case is `replicate`, additionally one can provide a password to the redis instance here but we recommend that by adding the line below to the .envrc
+
+```env
+export REDIS_PWD=your-redis-pwd
+```
 
-Run only the mq-store-agent with the following.
+`--avro-codec-path` - tells the BSP agent, the relative path to the AVRO .avsc files in the repo, since the agent ships with the corresponding .avsc files this remains fixed unless stated otherwise explicitly with another codec
 
-```bash
-    docker pull ghcr.io/covalenthq/mq-store-agent:latest
-    docker run ghcr.io/covalenthq/mq-store-agent:latest
+`--binary-file-path` - tells the BSP agent if local copies of the block-replica objects being created are to be stored in a given local directory. Please make sure the path (& directory) pre-exists before passing this flag
+
+`--gcp-svc-account` - sets the full path to the .json credentials file to the service account for the gcp bucket where the objects can be persisted
+
+`--replica-bucket` - lets the BSP agent know the “bucket-name” for cloud storage of block replica specimens/results (currently only google cloud storege is supported)
+
+`--segment-length` - allows the BSP operator to configure the size of each uploaded object (AVRO compression containing as many as specified block specimens in a single uploaded object)
+
+`--eth-client` - specifies the ethereum client connection string used to make transactions to on proof-chain contract, the respective credentials to be able to write to the contract should be provided in the .envrc file as follows
+
+```env
+export ETH_PRIVATE_KEY=cef7c71eac8558cc2953a519f80f0cb2541e15a3b0760e848895a78fd842d5a5
 ```
 
-Use `docker-compose` to get all the necessary services along with to also get running along with the mq-store-agent with the following from root. The other services are -
+`--proof-chain-address` - specifies the address of the proof-chain contract that has been deployed for the CQT network (local ethereum network for this workflow).
+
+`--consumer-timeout` - specifies in how many seconds the BSP agent stops waiting for new messages from the redis pending queue for decode, pack, encode, proof, store and upload.
+
+## <span id="docker">Docker</span>
+
+Please install [docker and docker-compose](https://docs.docker.com/compose/install/).
+
+Employ `docker-compose` to get all the necessary services along with the BSP agent to also get running along with the following, from root. Add a .env.dev file (if needed) to accomodate the env vars. The other services are -
 
 1. redis-srv (Open source (BSD licensed), in-memory data structure store)
 1. redis-commander-web (Redis web management tool written in node.js)
@@ -135,18 +246,27 @@ Use `docker-compose` to get all the necessary services along with to also get ru
 
 ```bash
     cd mq-store-agent
-    docker-compose -f "docker-compose.yml" up --build --force-recreate --remove-orphans
+    docker-compose -f "docker-compose.yml" --env-file .env.dev up --build --remove-orphans --force-recreate --exit-code-from consumer
+```
+
+The docker image for this service can be found [here](https://github.com/covalenthq/mq-store-agent/pkgs/container/mq-store-agent)
+
+Run only the mq-store-agent with the following, though this will not work if the other services in the docker-compose.yml file aren't also initialized.
+
+```bash
+    docker pull ghcr.io/covalenthq/mq-store-agent:latest
+    docker run ghcr.io/covalenthq/mq-store-agent:latest --env-file .env.dev
 ```
 
-## Scripts
+## <span id="scripts">Scripts</span>
 
-![diagram](wow.jpeg)
+![diagram](./docs/wow.jpeg)
 
 There are two lua scripts in `/scripts` for usage with the `redis-cli`.
 
 1. redis-count.lua - This allows for counting of total stream messages within bounds.
 
-    -- call with params [stream-key] , [first-stream-id] [last-stream-id] 
+    -- call with params [stream-key] , [first-stream-id] [last-stream-id]
     -- get to the ids with XINFO STREAM [stream-key]
 
 ```bash
@@ -164,15 +284,15 @@ There are two lua scripts in `/scripts` for usage with the `redis-cli`.
 > (integer) 5
 ```
 
-### Inspect
+### <span id="inspect">Inspect</span>
 
 To view pretty print the results from the creation of avro encoded block-replica files
 
 ```bash
 go run extractor.go \ 
-    --binary-file-path="../bin/block-replica/" \ 
-    --codec-path="../codec/block-replica.avsc" \ 
+    --binary-file-path="../bin/block-ethereum/" \
+    --avro-codec-path="../codec/block-ethereum.avsc" \
     --indent-json=0
 ```
 
-Please make sure that the --binary-file-path and --codec-path matches the ones given while running the agent above. --indent-json (0,1,2) can be used to pretty print and inspect the AVRO json objects.
+Please make sure that the --binary-file-path and --avro-codec-path matches the ones given while running the agent above. --indent-json (0,1,2) can be used to pretty print and inspect the AVRO json objects.
diff --git a/arch.png b/arch.png
diff --git a/cmd/mqstoreagent/main.go b/cmd/mqstoreagent/main.go
@@ -40,8 +40,8 @@ var (
 
 	// env flag vars
 	consumerPendingTimeoutFlag = 60 // defaults to 1 min
-	segmentLengthFlag          = 5  // defaults to 5 blocks per segment
-	codecPathFlag              string
+	segmentLengthFlag          = 1  // defaults to 1 block per segment
+	avroCodecPathFlag          string
 	redisURLFlag               string
 	replicaBucketFlag          string
 	gcpSvcAccountFlag          string
@@ -73,7 +73,7 @@ func init() {
 
 func main() {
 	flag.StringVar(&redisURLFlag, "redis-url", utils.LookupEnvOrString("RedisURL", redisURLFlag), "redis consumer stream url")
-	flag.StringVar(&codecPathFlag, "codec-path", utils.LookupEnvOrString("CodecPath", codecPathFlag), "local path to AVRO .avsc files housing the specimen/result schemas")
+	flag.StringVar(&avroCodecPathFlag, "avro-codec-path", utils.LookupEnvOrString("CodecPath", avroCodecPathFlag), "local path to AVRO .avsc files housing the specimen/result schemas")
 	flag.StringVar(&binaryFilePathFlag, "binary-file-path", utils.LookupEnvOrString("BinaryFilePath", binaryFilePathFlag), "local path to AVRO encoded binary files that contain block-replicas")
 	flag.StringVar(&gcpSvcAccountFlag, "gcp-svc-account", utils.LookupEnvOrString("GcpSvcAccount", gcpSvcAccountFlag), "local path to google cloud platform service account auth file")
 	flag.StringVar(&replicaBucketFlag, "replica-bucket", utils.LookupEnvOrString("ReplicaBucket", replicaBucketFlag), "google cloud platform object store target for specimen")
@@ -106,7 +106,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("unable to get ethereum client from Eth client flag: %v", err)
 	}
-	replicaAvro, err := avro.ParseSchemaFile(codecPathFlag)
+	replicaAvro, err := avro.ParseSchemaFile(avroCodecPathFlag)
 	if err != nil {
 		log.Fatalf("unable to parse avro schema for specimen from codec path flag: %v", err)
 	}