wip

metacraft-labs · Feb 29, 2024 · b34a06e · b34a06e
1 parent 3adea3f
commit b34a06e
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 61 deletions.
diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -8,8 +8,8 @@ logFilters:
   - code: YN0007
     level: discard
 
-# nodeLinker: node-modules
-nodeLinker: pnp
+nodeLinker: node-modules
+# nodeLinker: pnp
 isolatedNixBuilds: ["classic-level", "sqlite3", "bcrypt", "bcrypto", "leveldown", "redis-commander", "secp256k1", "blake-hash", "@chainsafe/blst", "keccak", "@trufflesuite/bigint-buffer", "msgpackr-extract", "bufferutil", "bufferutil", "utf-8-validate", "leveldown"]
 
 plugins:

diff --git a/beacon-light-client/plonky2/circuits/Cargo.toml b/beacon-light-client/plonky2/circuits/Cargo.toml
@@ -21,6 +21,6 @@ num = { version = "0.4.0", default-features = false }
 log = "0.4.14"
 redis-work-queue = "0.1.3"
 futures-lite = "1"
-redis = { version = "0.23", features = ["json", "aio", "tokio-comp"] }
+redis = { version = "0.23", features = ["aio", "tokio-comp"] }
 typenum = "1.16.0"
 
diff --git a/beacon-light-client/plonky2/circuits_executables/Cargo.lock b/beacon-light-client/plonky2/circuits_executables/Cargo.lock
diff --git a/beacon-light-client/plonky2/circuits_executables/Cargo.toml b/beacon-light-client/plonky2/circuits_executables/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2021"
 circuits = { path = "../circuits" }
 num = { version = "0.4.0", features = ["serde"] }
 plonky2 = { git = "https://github.com/metacraft-labs/plonky2" }
-redis = { version = "0.23", features = ["json", "aio", "tokio-comp"] }
+redis = { version = "0.23", features = ["aio", "tokio-comp"] }
 serde = "1.0.164"
 serde_json = "1.0.96"
 anyhow = "1.0.71"

diff --git a/beacon-light-client/plonky2/circuits_executables/src/crud/common.rs b/beacon-light-client/plonky2/circuits_executables/src/crud/common.rs
@@ -24,7 +24,7 @@ use plonky2::{
         circuit_data::CircuitData, config::PoseidonGoldilocksConfig, proof::ProofWithPublicInputs,
     },
 };
-use redis::{aio::Connection, AsyncCommands, JsonAsyncCommands};
+use redis::{aio::Connection, AsyncCommands};
 use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer};
 
 use super::proof_storage::proof_storage::ProofStorage;
@@ -180,12 +180,11 @@ pub async fn fetch_validator_balance_input(
 }
 
 pub async fn fetch_final_layer_input(con: &mut Connection) -> Result<FinalCircuitInput> {
-    let result: String = con
-        .json_get(VALIDATOR_COMMITMENT_CONSTANTS.final_proof_input_key, "$")
+    let json: String = con
+        .get(VALIDATOR_COMMITMENT_CONSTANTS.final_proof_input_key)
         .await?;
-    let result_vec = &serde_json::from_str::<Vec<FinalCircuitInput>>(&result)?;
-    ensure!(!result_vec.is_empty(), "Could not fetch json object");
-    Ok(result_vec[0].clone())
+    let result = serde_json::from_str::<FinalCircuitInput>(&json)?;
+    Ok(result)
 }
 
 pub async fn save_balance_proof(
@@ -214,16 +213,16 @@ pub async fn save_balance_proof(
         .set_proof(proof_index, &proof.to_bytes())
         .await?;
 
-    con.json_set(
-        format!(
+    save_json_object(
+        con,
+        &format!(
             "{}:{}:{}",
             VALIDATOR_COMMITMENT_CONSTANTS
                 .balance_verification_proof_key
                 .to_owned(),
             level,
             index
         ),
-        "$",
         &balance_proof,
     )
     .await?;
@@ -243,11 +242,9 @@ pub async fn save_final_proof(
         proof: proof.to_bytes(),
     };
 
-    con.json_set(
-        VALIDATOR_COMMITMENT_CONSTANTS
-            .final_layer_proof_key
-            .to_owned(),
-        "$",
+    save_json_object(
+        con,
+        &VALIDATOR_COMMITMENT_CONSTANTS.final_layer_proof_key,
         &final_proof,
     )
     .await?;
@@ -370,15 +367,15 @@ pub async fn save_zero_validator_proof(
         .set_proof(proof_index, &proof.to_bytes())
         .await?;
 
-    con.json_set(
-        format!(
+    save_json_object(
+        con,
+        &format!(
             "{}:zeroes:{}",
             VALIDATOR_COMMITMENT_CONSTANTS
                 .validator_proof_key
                 .to_owned(),
             depth,
         ),
-        "$",
         &validator_proof,
     )
     .await?;
@@ -410,16 +407,16 @@ pub async fn save_validator_proof(
         .set_proof(proof_index, &proof.to_bytes())
         .await?;
 
-    con.json_set(
-        format!(
+    save_json_object(
+        con,
+        &format!(
             "{}:{}:{}",
             VALIDATOR_COMMITMENT_CONSTANTS
                 .validator_proof_key
                 .to_owned(),
             gindex,
             epoch
         ),
-        "$",
         &validator_proof,
     )
     .await?;
@@ -454,10 +451,19 @@ pub async fn fetch_redis_json_object<T: DeserializeOwned + Clone>(
     con: &mut Connection,
     key: String,
 ) -> Result<T> {
-    let result: String = con.json_get(key, "$").await?;
-    let result_vec = &serde_json::from_str::<Vec<T>>(&result)?;
-    ensure!(!result_vec.is_empty(), "Could not fetch json object");
-    Ok(result_vec[0].clone())
+    let json: String = con.get(key).await?;
+    let result = serde_json::from_str::<T>(&json)?;
+    Ok(result)
+}
+
+pub async fn save_json_object<T: Serialize>(
+    con: &mut Connection,
+    key: &str,
+    object: &T,
+) -> Result<()> {
+    let json = serde_json::to_string(object)?;
+    con.set(key, json).await?;
+    Ok(())
 }
 
 pub async fn fetch_proof<T: NeedsChange + KeyProvider + DeserializeOwned + Clone>(

diff --git a/beacon-light-client/plonky2/tsconfig.json b/beacon-light-client/plonky2/tsconfig.json
@@ -1,3 +1,6 @@
 {
   "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "strictPropertyInitialization": false
+  }
 }
diff --git a/relay/implementations/redis.ts b/relay/implementations/redis.ts
@@ -11,7 +11,7 @@ import {
 } from '../types/types';
 import { RedisClientType, createClient } from 'redis';
 import CONSTANTS from '../../beacon-light-client/plonky2/constants/validator_commitment_constants.json';
-import { Redis as RedisClient} from 'ioredis';
+import { Redis as RedisClient } from 'ioredis';
 import chalk from 'chalk';
 
 export class Redis implements IRedis {
@@ -88,12 +88,21 @@ export class Redis implements IRedis {
     const latestEpoch = await this.getLatestEpoch(`${CONSTANTS.validatorProofKey}:${gindex}`, BigInt(epoch));
     if (latestEpoch === null) {
       const depth = Math.floor(Math.log2(Number(gindex) + 1));
-      const result = await this.client.json_get(`${CONSTANTS.validatorProofKey}:zeroes:${depth}`, hashKey) as any;
-      return result;
+      const result = await this.client.get(`${CONSTANTS.validatorProofKey}:zeroes:${depth}`);
+      if (result === null) {
+        return null;
+      }
+
+      return JSON.parse(result)[hashKey];
+
     }
 
     const key = `${CONSTANTS.validatorProofKey}:${gindex}:${latestEpoch}`;
-    return this.client.json_get(key, hashKey) as any;
+    const result = await this.client.get(key);
+    if (result === null) {
+      return null;
+    }
+    return JSON.parse(result)[hashKey];
   }
 
   async notifyAboutNewProof(): Promise<void> {
@@ -131,11 +140,11 @@ export class Redis implements IRedis {
     let allValidators: Validator[] = new Array(keys.length);
 
     for (const [keyBatchIndex, batchKeys] of splitIntoBatches(keys, batchSize).entries()) {
-      const res = await this.client.json_mget(batchKeys, '$');
+      const res = await this.client.mget(batchKeys);
       if (res === null) {
         continue;
       }
-      const batchValidators = (res as any[]).filter((v) => v !== null).flat();
+      const batchValidators = (res.filter((v) => v !== null) as string[]).map((json: any) => JSON.parse(json));
 
       for (const [index, redisValidator] of batchValidators.entries()) {
         try {
@@ -199,10 +208,7 @@ export class Redis implements IRedis {
   async isZeroBalanceEmpty() {
     await this.waitForConnection();
 
-    const result = await this.client.json_get(
-      `${CONSTANTS.validatorBalanceInputKey}:${CONSTANTS.validatorRegistryLimit}`,
-    );
-
+    const result = await this.client.get(`${CONSTANTS.validatorBalanceInputKey}:${CONSTANTS.validatorRegistryLimit}`);
     return result == null;
   }
 
@@ -213,12 +219,11 @@ export class Redis implements IRedis {
       await this.addToEpochLookup(`${CONSTANTS.validatorKey}:${validator.index}`, epoch);
       return [
         `${CONSTANTS.validatorKey}:${validator.index}:${epoch}`,
-        '$',
         JSON.stringify(validator.data),
       ];
     }));
 
-    await this.client.sendCommand(new RedisReJSON.Command('JSON.MSET', args));
+    await this.client.mset(args);
   }
 
   async saveValidatorBalancesInput(
@@ -229,12 +234,11 @@ export class Redis implements IRedis {
     const args = inputsWithIndices.map(ii => {
       return [
         `${CONSTANTS.validatorBalanceInputKey}:${ii.index}`,
-        '$',
         JSON.stringify(ii.input),
       ];
     });
 
-    await this.client.sendCommand(new RedisReJSON.Command('JSON.MSET', args));
+    await this.client.mset(args);
   }
 
   async saveFinalProofInput(input: {
@@ -248,10 +252,9 @@ export class Redis implements IRedis {
   }) {
     await this.waitForConnection();
 
-    await this.client.json_set(
+    await this.client.set(
       CONSTANTS.finalProofInputKey,
-      "$",
-      input as any
+      JSON.stringify(input),
     );
   }
 
@@ -266,7 +269,7 @@ export class Redis implements IRedis {
     },
   ): Promise<void> {
     await this.waitForConnection();
-    await this.client.json_set(`${CONSTANTS.validatorProofKey}:${gindex}:${epoch}`, "$", proof as any);
+    await this.client.set(`${CONSTANTS.validatorProofKey}:${gindex}:${epoch}`, JSON.stringify(proof));
   }
 
   async saveZeroValidatorProof(
@@ -279,7 +282,7 @@ export class Redis implements IRedis {
     },
   ): Promise<void> {
     await this.waitForConnection();
-    await this.client.json_set(`${CONSTANTS.validatorProofKey}:zeroes:${depth}`, "$", proof as any);
+    await this.client.set(`${CONSTANTS.validatorProofKey}:zeroes:${depth}`, JSON.stringify(proof));
   }
 
   async saveBalanceProof(
@@ -297,10 +300,9 @@ export class Redis implements IRedis {
   ): Promise<void> {
     await this.waitForConnection();
 
-    await this.client.json_set(
+    await this.client.set(
       `${CONSTANTS.balanceVerificationProofKey}:${level}:${index}`,
-      '$',
-      proof as any,
+      JSON.stringify(proof),
     );
   }
 
@@ -363,10 +365,9 @@ export class Redis implements IRedis {
   ): Promise<void> {
     await this.waitForConnection();
 
-    await this.client.json_set(
+    await this.client.set(
       `proof:${prevSlot}:${nextSlot}`,
-      '$',
-      proof as any,
+      JSON.stringify(proof),
     );
   }
 

diff --git a/yarn-project.nix b/yarn-project.nix
@@ -130,6 +130,11 @@ let
         "blake-hash@npm:2.0.0" \
         ${isolated."blake-hash@npm:2.0.0"} \
         ".yarn/unplugged/blake-hash-npm-2.0.0-c63b9a2c2d/node_modules/blake-hash"
+      echo 'injecting build for bcrypt'
+      yarn nixify inject-build \
+        "bcrypt@npm:5.0.1" \
+        ${isolated."bcrypt@npm:5.0.1"} \
+        ".yarn/unplugged/bcrypt-npm-5.0.1-6815be1cfe/node_modules/bcrypt"
       echo 'injecting build for bcrypto'
       yarn nixify inject-build \
         "bcrypto@npm:5.4.0" \
@@ -185,11 +190,6 @@ let
         "utf-8-validate@npm:5.0.10" \
         ${isolated."utf-8-validate@npm:5.0.10"} \
         ".yarn/unplugged/utf-8-validate-npm-5.0.10-93e9b6f750/node_modules/utf-8-validate"
-      echo 'injecting build for bcrypt'
-      yarn nixify inject-build \
-        "bcrypt@npm:5.0.1" \
-        ${isolated."bcrypt@npm:5.0.1"} \
-        ".yarn/unplugged/bcrypt-npm-5.0.1-6815be1cfe/node_modules/bcrypt"
       echo 'injecting build for redis-commander'
       yarn nixify inject-build \
         "redis-commander@npm:0.8.0" \
@@ -254,6 +254,7 @@ let
 
 isolated."msgpackr-extract@npm:3.0.0" = optionalOverride (args.overrideMsgpackrExtractAttrs or null) (mkIsolatedBuild { pname = "msgpackr-extract"; version = "3.0.0"; reference = "npm:3.0.0"; });
 isolated."blake-hash@npm:2.0.0" = optionalOverride (args.overrideBlakeHashAttrs or null) (mkIsolatedBuild { pname = "blake-hash"; version = "2.0.0"; reference = "npm:2.0.0"; });
+isolated."bcrypt@npm:5.0.1" = optionalOverride (args.overrideBcryptAttrs or null) (mkIsolatedBuild { pname = "bcrypt"; version = "5.0.1"; reference = "npm:5.0.1"; });
 isolated."bcrypto@npm:5.4.0" = optionalOverride (args.overrideBcryptoAttrs or null) (mkIsolatedBuild { pname = "bcrypto"; version = "5.4.0"; reference = "npm:5.4.0"; });
 isolated."keccak@npm:3.0.3" = optionalOverride (args.overrideKeccakAttrs or null) (mkIsolatedBuild { pname = "keccak"; version = "3.0.3"; reference = "npm:3.0.3"; });
 isolated."secp256k1@npm:4.0.3" = optionalOverride (args.overrideSecp256k1Attrs or null) (mkIsolatedBuild { pname = "secp256k1"; version = "4.0.3"; reference = "npm:4.0.3"; });
@@ -265,6 +266,5 @@ isolated."utf-8-validate@npm:5.0.7" = optionalOverride (args.overrideUtf8Validat
 isolated."msgpackr-extract@npm:3.0.2" = optionalOverride (args.overrideMsgpackrExtractAttrs or null) (mkIsolatedBuild { pname = "msgpackr-extract"; version = "3.0.2"; reference = "npm:3.0.2"; });
 isolated."bufferutil@npm:4.0.7" = optionalOverride (args.overrideBufferutilAttrs or null) (mkIsolatedBuild { pname = "bufferutil"; version = "4.0.7"; reference = "npm:4.0.7"; });
 isolated."utf-8-validate@npm:5.0.10" = optionalOverride (args.overrideUtf8ValidateAttrs or null) (mkIsolatedBuild { pname = "utf-8-validate"; version = "5.0.10"; reference = "npm:5.0.10"; });
-isolated."bcrypt@npm:5.0.1" = optionalOverride (args.overrideBcryptAttrs or null) (mkIsolatedBuild { pname = "bcrypt"; version = "5.0.1"; reference = "npm:5.0.1"; });
 isolated."redis-commander@npm:0.8.0" = optionalOverride (args.overrideRedisCommanderAttrs or null) (mkIsolatedBuild { pname = "redis-commander"; version = "0.8.0"; reference = "npm:0.8.0"; });
 in overriddenProject