Add integration testing with geth (#221)

Introduce a new crate `integration-tests` to do integration tests of the
rest of the crates with geth.  This crate contains a binary used to
generate data for the geth dev blockchain, and groups of integration
tests.  For now we only have the `rpc` test group.  See
`integration-tests/README.md` for more details.

Add a github action workflow that runs the integration test for the
bus-mapping rpc methods.  The action runs the `run.sh` integration test
script by steps to make it easier to see how long steps take, and to
quickly find which step fails in case of an error.

bus-mapping: Add `get_code_by_address` rpc method.

bus-mapping: Set the fields in the types returned by `get_proof` public,
and move them to `eth_types`.

Co-authored by: NoCtrlZ <phantomofrotten@gmail.com>

Author: ed255

.github/workflows/ci.yml

@@ -1,10 +1,10 @@
 name: CI checks
 
-on: 
+on:
   pull_request:
     types: [synchronize, opened, reopened, ready_for_review]
 
-## `actions-rs/toolchain@v1` overwrite set to false so that 
+## `actions-rs/toolchain@v1` overwrite set to false so that
 ## `rust-toolchain` is always used and the only source of truth.
 
 jobs:
@@ -28,7 +28,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --verbose --release --all --all-features
+          args: --verbose --release --all --all-features --exclude integration-tests
 
   build:
     if: github.event.pull_request.draft == false
@@ -98,7 +98,7 @@ jobs:
 
   fmt:
     if: github.event.pull_request.draft == false
-    
+
     name: Rustfmt
     timeout-minutes: 30
     runs-on: ubuntu-latest

.github/workflows/integration.yml

@@ -0,0 +1,36 @@
+name: Integration Tests
+
+on:
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  integration-tests:
+    if: github.event.pull_request.draft == false
+
+    name: Integration Tests
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ./integration-tests
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: false
+      - name: Set PATH
+        run: echo "${HOME}/bin" >> $GITHUB_PATH
+      - name: Install Solc
+        run: |
+            mkdir -p "$HOME/bin"
+            wget -q https://github.com/ethereum/solidity/releases/download/v0.8.0/solc-static-linux -O $HOME/bin/solc
+            chmod u+x "$HOME/bin/solc"
+            solc --version
+      # Run an initial build in a sepparate step to split the build time from execution time
+      - name: Build gendata bin
+        run: cargo build --bin gen_blockchain_data
+      - run: ./run.sh --steps "setup"
+      - run: ./run.sh --steps "gendata"
+      - run: ./run.sh --steps "tests" --tests "rpc"
+      - run: ./run.sh --steps "cleanup"

.github/workflows/lints.yml

@@ -2,14 +2,14 @@
 name: Lints
 
 # We only run these lints on trial-merges of PRs to reduce noise.
-on: 
+on:
   pull_request:
     types: [synchronize, opened, reopened, ready_for_review]
 
 jobs:
   clippy:
     if: github.event.pull_request.draft == false
-    
+
     name: Clippy
     timeout-minutes: 30
     runs-on: ubuntu-latest

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "bus-mapping",
     "keccak256",
     "geth-utils",
+    "integration-tests",
 ]
 
 [patch.crates-io]

bus-mapping/Cargo.toml

@@ -14,8 +14,8 @@ serde_json = "1.0.66"
 hex = "0.4"
 geth-utils = { path = "../geth-utils" }
 uint = "0.9.1"
-ethers-providers = "0.6.1"
-ethers-core = "0.6.1"
+ethers-providers = "0.6.2"
+ethers-core = "0.6.2"
 regex = "1.5.4"
 
 [dev-dependencies]

bus-mapping/src/eth_types.rs

@@ -4,8 +4,8 @@ use crate::evm::{memory::Memory, stack::Stack, storage::Storage};
 use crate::evm::{Gas, GasCost, OpcodeId, ProgramCounter};
 use ethers_core::types;
 pub use ethers_core::types::{
-    transaction::response::Transaction, Address, Block, Bytes,
-    EIP1186ProofResponse, H160, H256, U256, U64,
+    transaction::response::Transaction, Address, Block, Bytes, H160, H256,
+    U256, U64,
 };
 use pairing::arithmetic::FieldExt;
 use serde::{de, Deserialize};
@@ -133,6 +133,37 @@ impl<F: FieldExt> ToScalar<F> for Address {
     }
 }
 
+/// Struct used to define the storage proof
+#[derive(Debug, Default, Clone, PartialEq, Deserialize)]
+pub struct StorageProof {
+    /// Storage key
+    pub key: U256,
+    /// Storage Value
+    pub value: U256,
+    /// Storage proof: rlp-encoded trie nodes from root to value.
+    pub proof: Vec<Bytes>,
+}
+
+/// Struct used to define the result of `eth_getProof` call
+#[derive(Debug, Default, Clone, PartialEq, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct EIP1186ProofResponse {
+    /// Account address
+    pub address: Address,
+    /// The balance of the account
+    pub balance: U256,
+    /// The hash of the code of the account
+    pub code_hash: H256,
+    /// The nonce of the account
+    pub nonce: U256,
+    /// SHA3 of the StorageRoot
+    pub storage_hash: H256,
+    /// Array of rlp-serialized MerkleTree-Nodes
+    pub account_proof: Vec<Bytes>,
+    /// Array of storage-entries as requested
+    pub storage_proof: Vec<StorageProof>,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq, Deserialize)]
 #[doc(hidden)]
 struct GethExecStepInternal {

bus-mapping/src/rpc.rs

@@ -2,11 +2,12 @@
 //! query a Geth node in order to get a Block, Tx or Trace info.
 
 use crate::eth_types::{
-    Address, Block, EIP1186ProofResponse, GethExecTrace, Hash,
+    Address, Block, Bytes, EIP1186ProofResponse, GethExecTrace, Hash,
     ResultGethExecTraces, Transaction, Word, U64,
 };
 use crate::Error;
 use ethers_providers::JsonRpcClient;
+use serde::{Serialize, Serializer};
 
 /// Serialize a type.
 ///
@@ -22,7 +23,7 @@ pub fn serialize<T: serde::Serialize>(t: &T) -> serde_json::Value {
 #[derive(Debug)]
 pub enum BlockNumber {
     /// Specific block number
-    Num(U64),
+    Num(u64),
     /// Earliest block
     Earliest,
     /// Latest block
@@ -33,26 +34,27 @@ pub enum BlockNumber {
 
 impl From<u64> for BlockNumber {
     fn from(num: u64) -> Self {
-        BlockNumber::Num(U64::from(num))
+        BlockNumber::Num(num)
     }
 }
 
-impl BlockNumber {
-    /// Serializes a BlockNumber as a [`Value`](serde_json::Value) to be able to
-    /// throw it into a JSON-RPC request.
-    pub fn serialize(self) -> serde_json::Value {
+impl Serialize for BlockNumber {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
         match self {
-            BlockNumber::Num(num) => serialize(&num),
-            BlockNumber::Earliest => serialize(&"earliest"),
-            BlockNumber::Latest => serialize(&"latest"),
-            BlockNumber::Pending => serialize(&"pending"),
+            BlockNumber::Num(num) => U64::from(*num).serialize(serializer),
+            BlockNumber::Earliest => "earliest".serialize(serializer),
+            BlockNumber::Latest => "latest".serialize(serializer),
+            BlockNumber::Pending => "pending".serialize(serializer),
         }
     }
 }
 
 /// Placeholder structure designed to contain the methods that the BusMapping
 /// needs in order to enable Geth queries.
-pub struct GethClient<P: JsonRpcClient>(P);
+pub struct GethClient<P: JsonRpcClient>(pub P);
 
 impl<P: JsonRpcClient> GethClient<P> {
     /// Generates a new `GethClient` instance.
@@ -81,7 +83,7 @@ impl<P: JsonRpcClient> GethClient<P> {
         &self,
         block_num: BlockNumber,
     ) -> Result<Block<Transaction>, Error> {
-        let num = block_num.serialize();
+        let num = serialize(&block_num);
         let flag = serialize(&true);
         self.0
             .request("eth_getBlockByNumber", [num, flag])
@@ -112,7 +114,7 @@ impl<P: JsonRpcClient> GethClient<P> {
         &self,
         block_num: BlockNumber,
     ) -> Result<Vec<GethExecTrace>, Error> {
-        let num = block_num.serialize();
+        let num = serialize(&block_num);
         let resp: ResultGethExecTraces = self
             .0
             .request("debug_traceBlockByNumber", [num])
@@ -121,9 +123,25 @@ impl<P: JsonRpcClient> GethClient<P> {
         Ok(resp.0.into_iter().map(|step| step.result).collect())
     }
 
-    /// Calls `eth_getProof` via JSON-RPC returning a [`EIP1186ProofResponse`]
-    /// returning the account and storage-values of the specified
-    /// account including the Merkle-proof.
+    /// Calls `eth_getCode` via JSON-RPC returning a contract code
+    pub async fn get_code_by_address(
+        &self,
+        contract_address: Address,
+        block_num: BlockNumber,
+    ) -> Result<Vec<u8>, Error> {
+        let address = serialize(&contract_address);
+        let num = serialize(&block_num);
+        let resp: Bytes = self
+            .0
+            .request("eth_getCode", [address, num])
+            .await
+            .map_err(|e| Error::JSONRpcError(e.into()))?;
+        Ok(resp.to_vec())
+    }
+
+    /// Calls `eth_getProof` via JSON-RPC returning a
+    /// [`EIP1186ProofResponse`] returning the account and
+    /// storage-values of the specified account including the Merkle-proof.
     pub async fn get_proof(
         &self,
         account: Address,
@@ -132,130 +150,12 @@ impl<P: JsonRpcClient> GethClient<P> {
     ) -> Result<EIP1186ProofResponse, Error> {
         let account = serialize(&account);
         let keys = serialize(&keys);
-        let num = block_num.serialize();
+        let num = serialize(&block_num);
         self.0
             .request("eth_getProof", [account, keys, num])
             .await
             .map_err(|e| Error::JSONRpcError(e.into()))
     }
 }
 
-#[cfg(test)]
-mod rpc_tests {
-    use super::*;
-    use ethers_providers::Http;
-    use std::str::FromStr;
-    use url::Url;
-
-    // The test is ignored as the values used depend on the Geth instance used
-    // each time you run the tests. And we can't assume that everyone will
-    // have a Geth client synced with mainnet to have unified "test-vectors".
-    #[ignore]
-    #[tokio::test]
-    async fn test_get_block_by_hash() {
-        let transport = Http::new(Url::parse("http://localhost:8545").unwrap());
-
-        let hash = Hash::from_str("0xe4f7aa19a76fcf31a6adff3b400300849e39dd84076765fb3af09d05ee9d787a").unwrap();
-        let prov = GethClient::new(transport);
-        let block_by_hash = prov.get_block_by_hash(hash).await.unwrap();
-        assert!(hash == block_by_hash.hash.unwrap());
-    }
-
-    // The test is ignored as the values used depend on the Geth instance used
-    // each time you run the tests. And we can't assume that everyone will
-    // have a Geth client synced with mainnet to have unified "test-vectors".
-    #[ignore]
-    #[tokio::test]
-    async fn test_get_block_by_number() {
-        let transport = Http::new(Url::parse("http://localhost:8545").unwrap());
-
-        let hash = Hash::from_str("0xe4f7aa19a76fcf31a6adff3b400300849e39dd84076765fb3af09d05ee9d787a").unwrap();
-        let prov = GethClient::new(transport);
-        let block_by_num_latest =
-            prov.get_block_by_number(BlockNumber::Latest).await.unwrap();
-        assert!(hash == block_by_num_latest.hash.unwrap());
-        let block_by_num = prov.get_block_by_number(1u64.into()).await.unwrap();
-        assert!(
-            block_by_num.transactions[0].hash
-                == block_by_num_latest.transactions[0].hash
-        );
-    }
-
-    // The test is ignored as the values used depend on the Geth instance used
-    // each time you run the tests. And we can't assume that everyone will
-    // have a Geth client synced with mainnet to have unified "test-vectors".
-    #[ignore]
-    #[tokio::test]
-    async fn test_trace_block_by_hash() {
-        let transport = Http::new(Url::parse("http://localhost:8545").unwrap());
-
-        let hash = Hash::from_str("0xe2d191e9f663a3a950519eadeadbd614965b694a65a318a0b8f053f2d14261ff").unwrap();
-        let prov = GethClient::new(transport);
-        let trace_by_hash = prov.trace_block_by_hash(hash).await.unwrap();
-        // Since we called in the test block the same transaction twice the len
-        // should be the same and != 0.
-        assert!(
-            trace_by_hash[0].struct_logs.len()
-                == trace_by_hash[1].struct_logs.len()
-        );
-        assert!(!trace_by_hash[0].struct_logs.is_empty());
-    }
-
-    // The test is ignored as the values used depend on the Geth instance used
-    // each time you run the tests. And we can't assume that everyone will
-    // have a Geth client synced with mainnet to have unified "test-vectors".
-    #[ignore]
-    #[tokio::test]
-    async fn test_trace_block_by_number() {
-        let transport = Http::new(Url::parse("http://localhost:8545").unwrap());
-        let prov = GethClient::new(transport);
-        let trace_by_hash = prov.trace_block_by_number(5.into()).await.unwrap();
-        // Since we called in the test block the same transaction twice the len
-        // should be the same and != 0.
-        assert!(
-            trace_by_hash[0].struct_logs.len()
-                == trace_by_hash[1].struct_logs.len()
-        );
-        assert!(!trace_by_hash[0].struct_logs.is_empty());
-    }
-
-    // The test is ignored as the values used depend on the Geth instance used
-    // each time you run the tests. And we can't assume that everyone will
-    // have a Geth client synced with mainnet to have unified "test-vectors".
-    #[ignore]
-    #[tokio::test]
-    async fn test_get_proof() {
-        let transport = Http::new(Url::parse("http://localhost:8545").unwrap());
-        let prov = GethClient::new(transport);
-
-        let address =
-            Address::from_str("0x7F0d15C7FAae65896648C8273B6d7E43f58Fa842")
-                .unwrap();
-        let keys = vec![Word::from_str("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()];
-        let proof = prov
-            .get_proof(address, keys, BlockNumber::Latest)
-            .await
-            .unwrap();
-        const TARGET_PROOF: &str = r#"{
-            "address": "0x7f0d15c7faae65896648c8273b6d7e43f58fa842",
-            "accountProof": [
-                "0xf873a12050fb4d3174ec89ef969c09fd4391602169760fb005ad516f5d172cbffb80e955b84ff84d8089056bc75e2d63100000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
-            ],
-            "balance": "0x0",
-            "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-            "nonce": "0x0",
-            "storageHash": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
-            "storageProof": [
-                {
-                    "key": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
-                    "value": "0x0",
-                    "proof": []
-                }
-            ]
-        }"#;
-        assert!(
-            serde_json::from_str::<EIP1186ProofResponse>(TARGET_PROOF).unwrap()
-                == proof
-        );
-    }
-}
+// Integration tests found in `integration-tests/tests/rpc.rs`.

integration-tests/.gitignore

@@ -0,0 +1 @@
+gendata_output.json