Hstaykov/refactor loops

vendor/zkllvm-metacraft-circuits/docker/Dockerfile

@@ -4,5 +4,6 @@ RUN echo 'deb [trusted=yes]  http://deb.nil.foundation/ubuntu/ all main' >>/etc/
 RUN apt update && apt -y upgrade
 RUN apt install -y zkllvm proof-producer
 RUN apt install -y vim libyaml-cpp-dev cmake clang-format libboost-all-dev
+RUN apt update && apt -y upgrade
 
 ENTRYPOINT [ "/bin/bash", "-l", "-c" ]

vendor/zkllvm-metacraft-circuits/docs/compute_shuffled_index.md

@@ -0,0 +1,18 @@
+# Compute shuffled index documentation.
+
+### *Implementation*
+   The implementation code of the circuit is under "DendrETH/vendor/zkllvm-metacraft-circuits/src/circuits_impl/compute_shuffled_index_impl.h". 
+   This code is used in both the compilation as circuit and as executable.
+
+### *Circuit build*
+   In order to build as circuit, we need an entry point marked with the `[[circuit]]` directive. This is done through a wrapper 
+   that uses the implementation code and resides in "DendrETH/vendor/zkllvm-metacraft-circuits/src/circuits/compute_shuffled_index.cpp". Since currently the Crypto3
+   library does not implement computation of sha256 on a byte buffer, we use a header only library for the sha256 computations, which has 
+   negative performance consequences.
+
+### *Executable build + tests*
+   The implementation code of the circuit is compiled as executable and tested against the input data from 
+   https://github.com/ethereum/consensus-spec-tests.git. The tests reside in "DendrETH/vendor/zkllvm-metacraft-circuits/src/tests/compute_shuffled_index_test/".
+   For convenience, we have a script that performs all necessary steps to run the test -> "DendrETH/vendor/zkllvm-metacraft-circuits/scripts/compile_and_run_tests.sh", which by default runs all tests. We can pass as argument to this script "compute_shuffled_index_test" which will only run the relevant tests. For example, run the script from the main project directory "DendrETH/vendor/zkllvm-metacraft-circuits" as follows:
+   `./scripts/compile_and_run_tests.sh compute_shuffled_index_test`
+   It is required that docker is installed on the machine that will run the tests.

vendor/zkllvm-metacraft-circuits/docs/verify_attestation_data_and_proof_finality.md

@@ -0,0 +1,16 @@
+# Verify attestation data and proof finality documentation.
+
+### *Implementation*
+   The implementation code of the circuit is under "DendrETH/vendor/zkllvm-metacraft-circuits/src/circuits_impl/verify_attestation_data_impl.h". 
+   This code is used in both the compilation as circuit and as executable.
+
+### *Circuit build*
+   In order to build as circuit, we need an entry point marked with the `[[circuit]]` directive. This is done through a wrapper 
+   that uses the implementation code and resides in "DendrETH/vendor/zkllvm-metacraft-circuits/src/circuits/verify_attestation_data.cpp". This file also contains the proof_finality logic.
+
+### *Executable build + tests*
+   The implementation code of the circuit is compiled as executable and tested against input data extracted from 
+   an Ethereum node. The tests reside in "DendrETH/vendor/zkllvm-metacraft-circuits/src/tests/verify_attestation_data_test".
+   For convenience, we have a script that performs all necessary steps to run the test -> "DendrETH/vendor/zkllvm-metacraft-circuits/scripts/compile_and_run_tests.sh", which by default runs all tests. We can pass as argument to this script "verify_attestation_data_test" which will only run the relevant tests. For example, run the script from the main project directory "DendrETH/vendor/zkllvm-metacraft-circuits" as follows:
+   `./scripts/compile_and_run_tests.sh verify_attestation_data_test`
+   It is required that docker is installed on the machine that will run the tests.

vendor/zkllvm-metacraft-circuits/docs/weigh_justification_and_finalization.md

@@ -0,0 +1,15 @@
+# Weigh justification and finalization documentation.
+
+### *Implementation*
+   The implementation code of the circuit is under "DendrETH/vendor/zkllvm-metacraft-circuits/src/circuits_impl/weigh_justification_and_finalization_impl.h". 
+   This code is used in both the compilation as circuit and as executable.
+
+### *Circuit build*
+   In order to build as circuit, we need an entry point marked with the `[[circuit]]` directive. This is done through a wrapper 
+   that uses the implementation code and resides in "DendrETH/vendor/zkllvm-metacraft-circuits/src/circuits/weigh_justification_and_finalization.cpp".
+
+### *Executable build + tests*
+   The implementation code of the circuit is compiled as executable and tested against input data extracted from an Ethereum node. The tests reside in "DendrETH/vendor/zkllvm-metacraft-circuits/src/tests/weigh_justification_and_finalization_test".
+   For convenience, we have a script that performs all necessary steps to run the test -> "DendrETH/vendor/zkllvm-metacraft-circuits/scripts/compile_and_run_tests.sh", which by default runs all tests. We can pass as argument to this script "weigh_justification_and_finalization_test" which will only run the relevant tests. For example, run the script from the main project directory "DendrETH/vendor/zkllvm-metacraft-circuits" as follows:
+   `./scripts/compile_and_run_tests.sh weigh_justification_and_finalization_test`
+   It is required that docker is installed on the machine that will run the tests.

vendor/zkllvm-metacraft-circuits/scripts/compile_and_run_tests.sh

@@ -9,9 +9,19 @@ echo "CURRENT_DIR = " $CURRENT_DIR
 cd $SCRIPT_DIR/../docker && docker build -t zcli:latest -f Dockerfile_zcli . \
                          && docker run -v $SCRIPT_DIR/../../:/DendrETH --user $(id -u ${USER}):$(id -g ${USER}) zcli:latest
 
-if [ ! -d $SCRIPT_DIR/../src/tests/verify_attestation_data_test/finalizer-data ]
-then 
-    git clone git@github.com:metacraft-labs/finalizer-data.git $SCRIPT_DIR/../src/tests/verify_attestation_data_test/finalizer-data
+
+FINALIZER_DATA_TEST_DIR=$SCRIPT_DIR/../src/tests/verify_attestation_data_test/finalizer-data
+
+if [ ! -d $FINALIZER_DATA_TEST_DIR ]
+then
+    git clone git@github.com:metacraft-labs/finalizer-data.git $FINALIZER_DATA_TEST_DIR
+fi
+
+CONSENSUS_SPEC_TEST_DATA_DIR=$SCRIPT_DIR/../../consensus-spec-tests
+
+if [ ! -d $CONSENSUS_SPEC_TEST_DATA_DIR ]
+then
+    git clone git@github.com:ethereum/consensus-spec-tests.git $CONSENSUS_SPEC_TEST_DATA_DIR
 fi
 
 cd $CURRENT_DIR
@@ -25,4 +35,4 @@ then
     bash ${SCRIPT_DIR}/docker_run.sh "make -C ${ZKLLVM_BUILD:-build} test "
 else
     bash ${SCRIPT_DIR}/docker_run.sh "ctest --test-dir ${ZKLLVM_BUILD:-build} -R ${@}"
-fi
+fi

vendor/zkllvm-metacraft-circuits/src/circuit_utils/base_types.h

@@ -77,7 +77,9 @@ struct CheckpointVariable {
 
 struct JustificationBitsVariable {
 
-    static_vector<bool, 4, true> bits;
+    static constexpr auto bits_size = 4;
+
+    static_vector<bool, bits_size, true> bits;
 
     constexpr JustificationBitsVariable(const std::array<bool, decltype(bits)::capacity> &init) {
         size_t i = 0;
@@ -87,7 +89,7 @@ struct JustificationBitsVariable {
     }
 
     constexpr JustificationBitsVariable() {
-        for (size_t i = 0; i < bits.size(); ++i) {
+        for (size_t i = 0; i < bits_size; ++i) {
             bits[i] = false;
         }
     }
@@ -109,8 +111,11 @@ struct JustificationBitsVariable {
         assert_in_executable(lower_bound >= 0);
         assert_in_executable(upper_bound_non_inclusive <= bits.size());
         bool result = true;
-        for (size_t i = lower_bound; i < upper_bound_non_inclusive; i++) {
-            result = result && bits[i];
+        for (size_t i = 0; i < bits_size; i++) {
+            auto pos = i + lower_bound;
+            if (pos < upper_bound_non_inclusive) {
+                result = result && bits[pos];
+            }
         }
         return result;
     }

vendor/zkllvm-metacraft-circuits/src/circuit_utils/circuit_byte_utils.h

@@ -22,14 +22,6 @@ namespace circuit_byte_utils {
         return arr;
     }
 
-    template<class InputIt, class OutputIt>
-    OutputIt copy(InputIt first, InputIt last, OutputIt d_first) {
-        for (; first != last; (void)++first, (void)++d_first)
-            *d_first = *first;
-
-        return d_first;
-    }
-
     template<typename T>
     Byte get_nth_byte(const T val, unsigned int n) {
         static_assert(std::is_integral_v<typename std::remove_reference_t<T>>, "T must be integral");
@@ -69,7 +61,9 @@ namespace circuit_byte_utils {
         static_assert(N <= InputSize);
         assert_true(N + offset <= InputSize);
         static_vector<Byte, N> ret;
-        copy(val.begin() + offset, val.begin() + offset + N, ret.begin());
+        for (size_t i = 0; i < N; i++) {
+            *(ret.begin() + i) = *(val.begin() + offset + i);
+        }
 
         return ret;
     }
@@ -78,7 +72,9 @@ namespace circuit_byte_utils {
     static_vector<Byte, N> expand(const static_vector<Byte, InputSize>& val) {
         static_assert(N >= InputSize);
         static_vector<Byte, N> ret;
-        copy(val.begin(), val.end(), ret.begin());
+        for (size_t i = 0; i < InputSize; i++) {
+            *(ret.begin() + i) = *(val.begin() + i);
+        }
 
         return ret;
     }
@@ -102,18 +98,18 @@ namespace circuit_byte_utils {
     }
 
     template<typename T, typename IteratorType, bool LittleEndian>
-    T bytes_to_int(IteratorType first_element, IteratorType last_element) {
+    T bytes_to_int(IteratorType begin, IteratorType end) {
         static_assert(std::is_integral_v<typename std::remove_reference_t<T>>, "T must be integral");
-        assert_in_executable(first_element + sizeof(T) <= last_element);
+        assert_in_executable(begin + sizeof(T) <= end);
 
         T result = 0;
         if constexpr (LittleEndian) {
-            for (unsigned int i = sizeof(T); i > 0; i--) {
-                result = (result << 8) + *(first_element + i - 1);
+            for (unsigned i = 0; i < sizeof(T); ++i) {
+                result = (result << 8) + *(begin + sizeof(T) - i - 1);
             }
         } else {
             for (unsigned i = 0; i < sizeof(T); ++i) {
-                result = (result << 8) + *(first_element + i);
+                result = (result << 8) + *(begin + i);
             }
         }
         return result;
@@ -220,6 +216,31 @@ namespace circuit_byte_utils {
         return hashed;
     }
 
+    // This is needed, because since version 17.0.4 of clang-zkllvm the variadic template generates "warning: loop not
+    // unrolled"
+    Bytes32 sha256_33(const Bytes32& bytes, Byte b) {
+        static constexpr auto SIZE = 33;
+        std::array<Byte, SIZE> buffer;
+        memcpy(&buffer[0], &bytes.content_[0], Bytes32::capacity);
+        memcpy(&buffer[Bytes32::capacity], &b, 1);
+        Bytes32 hashed;
+        picosha2::hash256(buffer.begin(), buffer.begin() + SIZE, hashed.begin(), hashed.end());
+        return hashed;
+    }
+
+    // This is needed, because since version 17.0.4 of clang-zkllvm the variadic template generates "warning: loop not
+    // unrolled"
+    Bytes32 sha256_37(const Bytes32& bytes, Byte b, const static_vector<Byte, 4>& end) {
+        static constexpr auto SIZE = 37;
+        std::array<Byte, SIZE> buffer;
+        memcpy(&buffer[0], &bytes.content_[0], Bytes32::capacity);
+        memcpy(&buffer[Bytes32::capacity], &b, 1);
+        memcpy(&buffer[Bytes32::capacity] + 1, &end.content_[0], std::remove_reference<decltype(end)>::type::capacity);
+        Bytes32 hashed;
+        picosha2::hash256(buffer.begin(), buffer.begin() + SIZE, hashed.begin(), hashed.end());
+        return hashed;
+    }
+
     Bytes32 sha256_pair(const Bytes32& child1, const Bytes32& child2) {
         return sha256(child1, child2);
     }

vendor/zkllvm-metacraft-circuits/src/circuits_impl/compute_shuffled_index_impl.h

@@ -8,18 +8,25 @@
 
 using namespace circuit_byte_utils;
 
+static constexpr uint64_t MAX_SHUFFLE_ROUND_COUNT = 90;
+
 uint64_t compute_shuffled_index_impl(uint64_t index, uint64_t index_count, Bytes32 seed, int shuffle_round_count) {
     assert_true(index < index_count);
+    assert_true(shuffle_round_count <= MAX_SHUFFLE_ROUND_COUNT);
 
     // Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf)
     // See the 'generalized domain' algorithm on page 3
-    for (Byte current_round = 0; current_round < shuffle_round_count; current_round++) {
+    for (Byte current_round = 0; current_round < MAX_SHUFFLE_ROUND_COUNT; current_round++) {
+        // Exit after shuffle_round_count iterations
+        if (current_round == shuffle_round_count) {
+            return index;
+        }
 
-        auto pivot = bytes_to_int<uint64_t>(take<8>(sha256(seed, current_round))) % index_count;
+        auto pivot = bytes_to_int<uint64_t>(take<8>(sha256_33(seed, current_round))) % index_count;
         uint64_t flip = (pivot + index_count - index) % index_count;
         auto position = std::max(index, flip);
 
-        Bytes32 seed_hash = sha256(seed, current_round, int_to_bytes(uint32_t(position / 256)));
+        Bytes32 seed_hash = sha256_37(seed, current_round, int_to_bytes(uint32_t(position / 256)));
         auto byte = seed_hash[(position % 256) / 8];
         auto bit = (byte >> (position % 8)) % 2;