Lr/double quorum (#3922)

* Initial commit

* WIP: adding epoch to proposal and vote data, not compiling yet

* Make it compile

* Adjust tests

* Add a test type for two stake tables for even and odd epochs

* Debugging

* Fix extended voting

* Try "in epoch transition" approach

* Continue debugging

* Use correct epoch with Membership

* Adjust tests and lints

* Adapt to variable stake table after merge

* Fix accidentally pulled bug in eQC rule

* Commit includes epoch for vote and proposal data types

* Prune dependencies (#3787)

* add new message types and gate outgoing messages

* Use the proper message for the proposal response

* Modify commit for `Leaf2` and `QuorumData2`

* Adjust tests

* Clean up debug traces

* Initial commit for double quorum

* Add TODO

* Next epoch nodes vote during epoch transition

* Form the second QC at the end of an epoch

* Allow early payload save but check that's it's the same

* Attach next epoch justify qc to proposals

* Validate the next epoch justify qc

* Test with more network types

* Fix fmt in tests

* Use real threshold in the tests based on an epoch

* Membership thresholds depend on an epoch

* Make sure epoch transition proposals include the next epoch QC

* Use epoch from vote and add more tests

* Adjust marketplace ver number

* Epochs without Marketplace and adjust tests

* fix merge

* Fixes after merge

* Fix vid share handling

* Submit transactions to the correct epoch

* Address review comments

* Use one lock to get two values

---------

Co-authored-by: Artemii Gerasimovich <artemii@espressosys.com>
Co-authored-by: ss-es <155648797+ss-es@users.noreply.github.com>

crates/example-types/src/storage_types.rs

@@ -20,7 +20,7 @@ use hotshot_types::{
     },
     event::HotShotAction,
     message::Proposal,
-    simple_certificate::{QuorumCertificate2, UpgradeCertificate},
+    simple_certificate::{NextEpochQuorumCertificate2, QuorumCertificate2, UpgradeCertificate},
     traits::{
         node_implementation::{ConsensusTime, NodeType},
         storage::Storage,
@@ -52,6 +52,8 @@ pub struct TestStorageState<TYPES: NodeType> {
     proposals2: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal2<TYPES>>>,
     high_qc: Option<hotshot_types::simple_certificate::QuorumCertificate<TYPES>>,
     high_qc2: Option<hotshot_types::simple_certificate::QuorumCertificate2<TYPES>>,
+    next_epoch_high_qc2:
+        Option<hotshot_types::simple_certificate::NextEpochQuorumCertificate2<TYPES>>,
     action: TYPES::View,
     epoch: TYPES::Epoch,
 }
@@ -66,6 +68,7 @@ impl<TYPES: NodeType> Default for TestStorageState<TYPES> {
             proposals: BTreeMap::new(),
             proposals2: BTreeMap::new(),
             high_qc: None,
+            next_epoch_high_qc2: None,
             high_qc2: None,
             action: TYPES::View::genesis(),
             epoch: TYPES::Epoch::genesis(),
@@ -112,6 +115,9 @@ impl<TYPES: NodeType> TestStorage<TYPES> {
     pub async fn high_qc_cloned(&self) -> Option<QuorumCertificate2<TYPES>> {
         self.inner.read().await.high_qc2.clone()
     }
+    pub async fn next_epoch_high_qc_cloned(&self) -> Option<NextEpochQuorumCertificate2<TYPES>> {
+        self.inner.read().await.next_epoch_high_qc2.clone()
+    }
     pub async fn decided_upgrade_certificate(&self) -> Option<UpgradeCertificate<TYPES>> {
         self.decided_upgrade_certificate.read().await.clone()
     }
@@ -268,6 +274,26 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
         }
         Ok(())
     }
+    async fn update_next_epoch_high_qc2(
+        &self,
+        new_next_epoch_high_qc: hotshot_types::simple_certificate::NextEpochQuorumCertificate2<
+            TYPES,
+        >,
+    ) -> Result<()> {
+        if self.should_return_err {
+            bail!("Failed to update next epoch high qc to storage");
+        }
+        Self::run_delay_settings_from_config(&self.delay_config).await;
+        let mut inner = self.inner.write().await;
+        if let Some(ref current_next_epoch_high_qc) = inner.next_epoch_high_qc2 {
+            if new_next_epoch_high_qc.view_number() > current_next_epoch_high_qc.view_number() {
+                inner.next_epoch_high_qc2 = Some(new_next_epoch_high_qc);
+            }
+        } else {
+            inner.next_epoch_high_qc2 = Some(new_next_epoch_high_qc);
+        }
+        Ok(())
+    }
     async fn update_undecided_state(
         &self,
         _leaves: CommitmentMap<Leaf<TYPES>>,

crates/hotshot/src/lib.rs

@@ -52,7 +52,7 @@ use hotshot_types::{
     data::{Leaf2, QuorumProposal, QuorumProposal2},
     event::{EventType, LeafInfo},
     message::{convert_proposal, DataMessage, Message, MessageKind, Proposal},
-    simple_certificate::{QuorumCertificate2, UpgradeCertificate},
+    simple_certificate::{NextEpochQuorumCertificate2, QuorumCertificate2, UpgradeCertificate},
     traits::{
         consensus_api::ConsensusApi,
         election::Membership,
@@ -344,6 +344,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
             saved_leaves,
             saved_payloads,
             initializer.high_qc,
+            initializer.next_epoch_high_qc,
             Arc::clone(&consensus_metrics),
             config.epoch_height,
         );
@@ -492,7 +493,11 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
         trace!("Adding transaction to our own queue");
 
         let api = self.clone();
-        let view_number = api.consensus.read().await.cur_view();
+
+        let consensus_reader = api.consensus.read().await;
+        let view_number = consensus_reader.cur_view();
+        let epoch = consensus_reader.cur_epoch();
+        drop(consensus_reader);
 
         // Wrap up a message
         let message_kind: DataMessage<TYPES> =
@@ -518,7 +523,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
                 api
                     .network.da_broadcast_message(
                         serialized_message,
-                        api.memberships.da_committee_members(view_number, TYPES::Epoch::new(1)).iter().cloned().collect(),
+                        api.memberships.da_committee_members(view_number, epoch).iter().cloned().collect(),
                         BroadcastDelay::None,
                     ),
                 api
@@ -1000,6 +1005,8 @@ pub struct HotShotInitializer<TYPES: NodeType> {
     /// than `inner`s view number for the non genesis case because we must have seen higher QCs
     /// to decide on the leaf.
     high_qc: QuorumCertificate2<TYPES>,
+    /// Next epoch highest QC that was seen. This is needed to propose during epoch transition after restart.
+    next_epoch_high_qc: Option<NextEpochQuorumCertificate2<TYPES>>,
     /// Previously decided upgrade certificate; this is necessary if an upgrade has happened and we are not restarting with the new version
     decided_upgrade_certificate: Option<UpgradeCertificate<TYPES>>,
     /// Undecided leaves that were seen, but not yet decided on.  These allow a restarting node
@@ -1030,6 +1037,7 @@ impl<TYPES: NodeType> HotShotInitializer<TYPES> {
             actioned_view: TYPES::View::new(0),
             saved_proposals: BTreeMap::new(),
             high_qc,
+            next_epoch_high_qc: None,
             decided_upgrade_certificate: None,
             undecided_leaves: Vec::new(),
             undecided_state: BTreeMap::new(),
@@ -1054,6 +1062,7 @@ impl<TYPES: NodeType> HotShotInitializer<TYPES> {
         actioned_view: TYPES::View,
         saved_proposals: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal2<TYPES>>>,
         high_qc: QuorumCertificate2<TYPES>,
+        next_epoch_high_qc: Option<NextEpochQuorumCertificate2<TYPES>>,
         decided_upgrade_certificate: Option<UpgradeCertificate<TYPES>>,
         undecided_leaves: Vec<Leaf2<TYPES>>,
         undecided_state: BTreeMap<TYPES::View, View<TYPES>>,
@@ -1068,6 +1077,7 @@ impl<TYPES: NodeType> HotShotInitializer<TYPES> {
             actioned_view,
             saved_proposals,
             high_qc,
+            next_epoch_high_qc,
             decided_upgrade_certificate,
             undecided_leaves,
             undecided_state,

crates/hotshot/src/tasks/task_state.rs

@@ -132,6 +132,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> CreateTaskState
             public_key: handle.public_key().clone(),
             private_key: handle.private_key().clone(),
             id: handle.hotshot.id,
+            epoch_height: handle.epoch_height,
         }
     }
 }
@@ -318,6 +319,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> CreateTaskState
             network: Arc::clone(&handle.hotshot.network),
             membership: (*handle.hotshot.memberships).clone().into(),
             vote_collectors: BTreeMap::default(),
+            next_epoch_vote_collectors: BTreeMap::default(),
             timeout_vote_collectors: BTreeMap::default(),
             cur_view: handle.cur_view().await,
             cur_view_time: Utc::now().timestamp(),

crates/hotshot/src/traits/election/static_committee.rs

@@ -215,22 +215,22 @@ impl<TYPES: NodeType> Membership<TYPES> for StaticCommittee<TYPES> {
     }
 
     /// Get the voting success threshold for the committee
-    fn success_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn success_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         NonZeroU64::new(((self.stake_table.len() as u64 * 2) / 3) + 1).unwrap()
     }
 
     /// Get the voting success threshold for the committee
-    fn da_success_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn da_success_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         NonZeroU64::new(((self.da_stake_table.len() as u64 * 2) / 3) + 1).unwrap()
     }
 
     /// Get the voting failure threshold for the committee
-    fn failure_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn failure_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         NonZeroU64::new(((self.stake_table.len() as u64) / 3) + 1).unwrap()
     }
 
     /// Get the voting upgrade threshold for the committee
-    fn upgrade_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn upgrade_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         let len = self.stake_table.len();
         NonZeroU64::new(max((len as u64 * 9) / 10, ((len as u64 * 2) / 3) + 1)).unwrap()
     }

crates/hotshot/src/traits/election/static_committee_leader_two_views.rs

@@ -217,22 +217,22 @@ impl<TYPES: NodeType> Membership<TYPES> for StaticCommitteeLeaderForTwoViews<TYP
     }
 
     /// Get the voting success threshold for the committee
-    fn success_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn success_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         NonZeroU64::new(((self.stake_table.len() as u64 * 2) / 3) + 1).unwrap()
     }
 
     /// Get the voting success threshold for the committee
-    fn da_success_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn da_success_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         NonZeroU64::new(((self.da_stake_table.len() as u64 * 2) / 3) + 1).unwrap()
     }
 
     /// Get the voting failure threshold for the committee
-    fn failure_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn failure_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         NonZeroU64::new(((self.stake_table.len() as u64) / 3) + 1).unwrap()
     }
 
     /// Get the voting upgrade threshold for the committee
-    fn upgrade_threshold(&self, _epoch: <TYPES as NodeType>::Epoch) -> NonZeroU64 {
+    fn upgrade_threshold(&self, _epoch: TYPES::Epoch) -> NonZeroU64 {
         NonZeroU64::new(((self.stake_table.len() as u64 * 9) / 10) + 1).unwrap()
     }
 }

crates/task-impls/src/consensus/handlers.rs

@@ -10,12 +10,13 @@ use async_broadcast::Sender;
 use chrono::Utc;
 use hotshot_types::{
     event::{Event, EventType},
-    simple_vote::{QuorumVote2, TimeoutData2, TimeoutVote2},
+    simple_vote::{HasEpoch, QuorumVote2, TimeoutData2, TimeoutVote2},
     traits::{
         election::Membership,
         node_implementation::{ConsensusTime, NodeImplementation, NodeType},
     },
-    vote::HasViewNumber,
+    utils::EpochTransitionIndicator,
+    vote::{HasViewNumber, Vote},
 };
 use tokio::{spawn, time::sleep};
 use tracing::instrument;
@@ -46,7 +47,7 @@ pub(crate) async fn handle_quorum_vote_recv<
         .is_high_qc_for_last_block();
     let we_are_leader = task_state
         .membership
-        .leader(vote.view_number() + 1, task_state.cur_epoch)?
+        .leader(vote.view_number() + 1, vote.data.epoch)?
         == task_state.public_key;
     ensure!(
         in_transition || we_are_leader,
@@ -56,20 +57,45 @@ pub(crate) async fn handle_quorum_vote_recv<
         )
     );
 
+    let transition_indicator = if in_transition {
+        EpochTransitionIndicator::InTransition
+    } else {
+        EpochTransitionIndicator::NotInTransition
+    };
     handle_vote(
         &mut task_state.vote_collectors,
         vote,
         task_state.public_key.clone(),
         &task_state.membership,
-        task_state.cur_epoch,
+        vote.data.epoch,
         task_state.id,
         &event,
         sender,
         &task_state.upgrade_lock,
-        !in_transition,
+        transition_indicator.clone(),
     )
     .await?;
 
+    // If the vote sender belongs to the next epoch, collect it separately to form the second QC
+    if task_state
+        .membership
+        .has_stake(&vote.signing_key(), vote.epoch() + 1)
+    {
+        handle_vote(
+            &mut task_state.next_epoch_vote_collectors,
+            &vote.clone().into(),
+            task_state.public_key.clone(),
+            &task_state.membership,
+            vote.data.epoch,
+            task_state.id,
+            &event,
+            sender,
+            &task_state.upgrade_lock,
+            transition_indicator,
+        )
+        .await?;
+    }
+
     Ok(())
 }
 
@@ -101,12 +127,12 @@ pub(crate) async fn handle_timeout_vote_recv<
         vote,
         task_state.public_key.clone(),
         &task_state.membership,
-        task_state.cur_epoch,
+        vote.data.epoch,
         task_state.id,
         &event,
         sender,
         &task_state.upgrade_lock,
-        true,
+        EpochTransitionIndicator::NotInTransition,
     )
     .await?;
 

crates/task-impls/src/consensus/mod.rs

@@ -14,8 +14,8 @@ use hotshot_types::{
     consensus::OuterConsensus,
     event::Event,
     message::UpgradeLock,
-    simple_certificate::{QuorumCertificate2, TimeoutCertificate2},
-    simple_vote::{QuorumVote2, TimeoutVote2},
+    simple_certificate::{NextEpochQuorumCertificate2, QuorumCertificate2, TimeoutCertificate2},
+    simple_vote::{NextEpochQuorumVote2, QuorumVote2, TimeoutVote2},
     traits::{
         node_implementation::{ConsensusTime, NodeImplementation, NodeType, Versions},
         signature_key::SignatureKey,
@@ -55,6 +55,14 @@ pub struct ConsensusTaskState<TYPES: NodeType, I: NodeImplementation<TYPES>, V:
     /// A map of `QuorumVote` collector tasks.
     pub vote_collectors: VoteCollectorsMap<TYPES, QuorumVote2<TYPES>, QuorumCertificate2<TYPES>, V>,
 
+    /// A map of `QuorumVote` collector tasks. They collect votes from the nodes in the next epoch.
+    pub next_epoch_vote_collectors: VoteCollectorsMap<
+        TYPES,
+        NextEpochQuorumVote2<TYPES>,
+        NextEpochQuorumCertificate2<TYPES>,
+        V,
+    >,
+
     /// A map of `TimeoutVote` collector tasks.
     pub timeout_vote_collectors:
         VoteCollectorsMap<TYPES, TimeoutVote2<TYPES>, TimeoutCertificate2<TYPES>, V>,