Make leader work no matter what the CWD is (#307)

* `prove_rpc.sh` now outputs to a constant dir

- Previously took into account the directory that the script was
  executed from.

* `prove_stdio.sh` now works when used from any directory

- Previously only worked when executed from the `tools` directory.

* Now always uses `tools/circuits` in workspace

- Previously, we always looked for the `circuits` directory directly inside our
  current working directory. Now if we are running within the workspace,
  we always will look in `zk_evm/zero_bin/tools` instead.
- Also note that `zero_bin/.cargo/config.toml` is only read when we are
  interacting with `cargo` directly (eg. the scripts in `tools/` do not
  invoke it).

* Now also writes other artifacts in `tools/`

- Also made sense to write `verify.out`, `leader.out` and `proofs.json`
  into `zero_bin/tools`.

* Made a few other artifacts also get written to `tools/`

- Also removed some duplication by moving them to variables.

zero_bin/.cargo/config.toml

@@ -2,3 +2,7 @@
 # https://github.com/rust-lang/rust/pull/124129
 # https://github.com/dtolnay/linkme/pull/88
 rustflags = ["-Zlinker-features=-lld"]
+
+[env]
+# If we're running in the project workspace, then we should set the workspace env var so we read/write to/from files relative to the workspace.
+CARGO_WORKSPACE_DIR = { value = "", relative = true }

zero_bin/common/src/prover_state/persistence.rs

@@ -17,9 +17,10 @@ use super::{
     Config, RecursiveCircuitsForTableSize, SIZE,
 };
 
-const CIRCUITS_FOLDER: &str = "./circuits";
+const CIRCUITS_DIR: &str = "circuits/";
 const PROVER_STATE_FILE_PREFIX: &str = "prover_state";
 const VERIFIER_STATE_FILE_PREFIX: &str = "verifier_state";
+const CARGO_WORKSPACE_DIR_ENV: &str = "CARGO_WORKSPACE_DIR";
 
 fn get_serializers() -> (
     DefaultGateSerializer,
@@ -72,9 +73,11 @@ pub(crate) trait DiskResource {
         p: &Self::PathConstrutor,
         r: &Self::Resource,
     ) -> Result<(), DiskResourceError<Self::Error>> {
+        let circuits_dir = relative_circuit_dir_path();
+
         // Create the base folder if non-existent.
-        if std::fs::metadata(CIRCUITS_FOLDER).is_err() {
-            std::fs::create_dir(CIRCUITS_FOLDER).map_err(|_| {
+        if std::fs::metadata(&circuits_dir).is_err() {
+            std::fs::create_dir(&circuits_dir).map_err(|_| {
                 DiskResourceError::IoError::<Self::Error>(std::io::Error::other(
                     "Could not create circuits folder",
                 ))
@@ -104,7 +107,7 @@ impl DiskResource for BaseProverResource {
     fn path(p: &Self::PathConstrutor) -> impl AsRef<Path> {
         format!(
             "{}/{}_base_{}_{}",
-            CIRCUITS_FOLDER,
+            &relative_circuit_dir_path(),
             PROVER_STATE_FILE_PREFIX,
             env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()),
             p.get_configuration_digest()
@@ -140,7 +143,7 @@ impl DiskResource for MonolithicProverResource {
     fn path(p: &Self::PathConstrutor) -> impl AsRef<Path> {
         format!(
             "{}/{}_monolithic_{}_{}",
-            CIRCUITS_FOLDER,
+            &relative_circuit_dir_path(),
             PROVER_STATE_FILE_PREFIX,
             env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()),
             p.get_configuration_digest()
@@ -175,7 +178,7 @@ impl DiskResource for RecursiveCircuitResource {
     fn path((circuit_type, size): &Self::PathConstrutor) -> impl AsRef<Path> {
         format!(
             "{}/{}_{}_{}_{}",
-            CIRCUITS_FOLDER,
+            &relative_circuit_dir_path(),
             PROVER_STATE_FILE_PREFIX,
             env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()),
             circuit_type.as_short_str(),
@@ -219,7 +222,7 @@ impl DiskResource for VerifierResource {
     fn path(p: &Self::PathConstrutor) -> impl AsRef<Path> {
         format!(
             "{}/{}_{}_{}",
-            CIRCUITS_FOLDER,
+            &relative_circuit_dir_path(),
             VERIFIER_STATE_FILE_PREFIX,
             env::var("EVM_ARITHMETIZATION_PKG_VER").unwrap_or("NA".to_string()),
             p.get_configuration_digest()
@@ -273,3 +276,12 @@ fn prover_to_disk(
 
     Ok(())
 }
+
+/// If we're running in the cargo workspace, then always use the `circuits`
+/// directory that lives in `tools/`. Otherwise, just use `circuits` in the
+/// current directory.
+fn relative_circuit_dir_path() -> String {
+    env::var(CARGO_WORKSPACE_DIR_ENV)
+        .map(|p| format!("{}/{}", p, CIRCUITS_DIR))
+        .unwrap_or_else(|_| CIRCUITS_DIR.to_string())
+}

zero_bin/tools/prove_rpc.sh

@@ -37,7 +37,13 @@ else
   export MEMORY_CIRCUIT_SIZE="17..28"
 fi
 
-PROOF_OUTPUT_DIR="proofs"
+# Force the working directory to always be the `tools/` directory. 
+TOOLS_DIR=$(dirname $(realpath "$0"))
+
+# Set the environment variable to let the binary know that we're running in the project workspace.
+export CARGO_WORKSPACE_DIR="${TOOLS_DIR}/../"
+
+PROOF_OUTPUT_DIR="${TOOLS_DIR}/proofs"
 OUT_LOG_PATH="${PROOF_OUTPUT_DIR}/b$1_$2.log"
 ALWAYS_WRITE_LOGS=0 # Change this to `1` if you always want logs to be written.
 TOT_BLOCKS=$(($2-$1+1))
@@ -57,7 +63,6 @@ RUN_VERIFICATION="${RUN_VERIFICATION:-false}"
 
 mkdir -p $PROOF_OUTPUT_DIR
 
-
 if [ $IGNORE_PREVIOUS_PROOFS ]; then
     # Set checkpoint height to previous block number for the first block in range
     prev_proof_num=$(($1-1))

zero_bin/tools/prove_stdio.sh

@@ -11,6 +11,17 @@
 # We're going to set the parallelism in line with the total cpu count
 num_procs=$(nproc)
 
+# Force the working directory to always be the `tools/` directory. 
+TOOLS_DIR=$(dirname $(realpath "$0"))
+
+LEADER_OUT_PATH="${TOOLS_DIR}/leader.out"
+PROOFS_JSON_PATH="${TOOLS_DIR}/proofs.json"
+VERIFY_OUT_PATH="${TOOLS_DIR}/verify.out"
+TEST_OUT_PATH="${TOOLS_DIR}/test.out"
+
+# Set the environment variable to let the binary know that we're running in the project workspace.
+export CARGO_WORKSPACE_DIR="${TOOLS_DIR}/../"
+
 # Configured Rayon and Tokio with rough defaults
 export RAYON_NUM_THREADS=$num_procs
 export TOKIO_WORKER_THREADS=$num_procs
@@ -67,27 +78,27 @@ fi
 # proof. This is useful for quickly testing decoding and all of the
 # other non-proving code.
 if [[ $TEST_ONLY == "test_only" ]]; then
-    cargo run --release --features test_only --bin leader -- --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee test.out
-    if grep -q 'All proof witnesses have been generated successfully.' test.out; then
+    cargo run --release --features test_only --bin leader -- --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee $TEST_OUT_PATH
+    if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then
         echo -e "\n\nSuccess - Note this was just a test, not a proof"
         exit
     else
-         echo "Failed to create proof witnesses. See test.out for more details."
+         echo "Failed to create proof witnesses. See \"zk_evm/tools/test.out\" for more details."
         exit 1
     fi
 fi
 
 cargo build --release --jobs "$num_procs"
 
 start_time=$(date +%s%N)
-../../target/release/leader --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee leader.out
+"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand stdio < $INPUT_FILE | tee $LEADER_OUT_PATH
 end_time=$(date +%s%N)
 
-tail -n 1 leader.out > proofs.json
+tail -n 1 $LEADER_OUT_PATH > $PROOFS_JSON_PATH
 
-../../target/release/verifier -f proofs.json | tee verify.out
+"${TOOLS_DIR}/../../target/release/verifier" -f $PROOFS_JSON_PATH | tee $VERIFY_OUT_PATH
 
-if grep -q 'All proofs verified successfully!' verify.out; then
+if grep -q 'All proofs verified successfully!' $VERIFY_OUT_PATH; then
     duration_ns=$((end_time - start_time))
     duration_sec=$(echo "$duration_ns / 1000000000" | bc -l)
     echo "Success!"