From 256d9042d35837740e05741b467afd6b23b476c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 15 Feb 2024 00:41:52 +0000 Subject: [PATCH 01/21] Drop gossipsub stale messages when polling ConnectionHandler. (#5175) * drop gossipsub stale messages * convert async-channel::Receiver to Peekable, to be able to Peek next message without dropping it --- .../src/gossipsub/behaviour/tests.rs | 843 ++++++++++-------- .../src/gossipsub/handler.rs | 32 +- .../lighthouse_network/src/gossipsub/types.rs | 110 ++- 3 files changed, 548 insertions(+), 437 deletions(-) diff --git a/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs b/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs index 9d8a10bcc85..eb006e52928 100644 --- a/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs +++ b/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs @@ -95,7 +95,7 @@ where // build and connect peer_no random peers let mut peers = vec![]; - let mut receiver_queues = HashMap::new(); + let mut receivers = HashMap::new(); let empty = vec![]; for i in 0..self.peer_no { @@ -110,10 +110,10 @@ where i < self.explicit, ); peers.push(peer); - receiver_queues.insert(peer, receiver); + receivers.insert(peer, receiver); } - (gs, peers, receiver_queues, topic_hashes) + (gs, peers, receivers, topic_hashes) } fn peer_no(mut self, peer_no: usize) -> Self { @@ -420,7 +420,7 @@ fn test_subscribe() { // - run JOIN(topic) let subscribe_topic = vec![String::from("test_subscribe")]; - let (gs, _, queues, topic_hashes) = inject_nodes1() + let (gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(subscribe_topic) .to_subscribe(true) @@ -432,11 +432,12 @@ fn test_subscribe() { ); // collect all the subscriptions - let subscriptions = queues + let subscriptions = receivers .into_values() .fold(0, |mut collected_subscriptions, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Subscribe(_)) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Subscribe(_)) = priority.try_recv() { collected_subscriptions += 1 } } @@ -447,8 +448,8 @@ fn test_subscribe() { assert_eq!(subscriptions, 20); } -#[test] /// Test unsubscribe. +#[test] fn test_unsubscribe() { // Unsubscribe should: // - Remove the mesh entry for topic @@ -462,7 +463,7 @@ fn test_unsubscribe() { .collect::>(); // subscribe to topic_strings - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(topic_strings) .to_subscribe(true) @@ -492,11 +493,12 @@ fn test_unsubscribe() { ); // collect all the subscriptions - let subscriptions = queues + let subscriptions = receivers .into_values() .fold(0, |mut collected_subscriptions, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Subscribe(_)) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Subscribe(_)) = priority.try_recv() { collected_subscriptions += 1 } } @@ -515,8 +517,8 @@ fn test_unsubscribe() { } } -#[test] /// Test JOIN(topic) functionality. +#[test] fn test_join() { // The Join function should: // - Remove peers from fanout[topic] @@ -540,7 +542,7 @@ fn test_join() { .create_network(); // Flush previous GRAFT messages. - flush_events(&mut gs, &receivers); + receivers = flush_events(&mut gs, receivers); // unsubscribe, then call join to invoke functionality assert!( @@ -564,17 +566,33 @@ fn test_join() { "Should have added 6 nodes to the mesh" ); - fn count_grafts(mut acc: usize, receiver: &RpcReceiver) -> usize { - while !receiver.priority.is_empty() || !receiver.non_priority.is_empty() { - if let Ok(RpcOut::Graft(_)) = receiver.priority.try_recv() { - acc += 1; + fn count_grafts( + receivers: HashMap, + ) -> (usize, HashMap) { + let mut new_receivers = HashMap::new(); + let mut acc = 0; + + for (peer_id, c) in receivers.into_iter() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Graft(_)) = priority.try_recv() { + acc += 1; + } } + new_receivers.insert( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: priority.peekable(), + non_priority: c.non_priority, + }, + ); } - acc + (acc, new_receivers) } // there should be mesh_n GRAFT messages. - let graft_messages = receivers.values().fold(0, count_grafts); + let (graft_messages, mut receivers) = count_grafts(receivers); assert_eq!( graft_messages, 6, @@ -645,12 +663,12 @@ fn test_join() { ); } - // there should now be 12 graft messages to be sent - let graft_messages = receivers.values().fold(graft_messages, count_grafts); + // there should now 6 graft messages to be sent + let (graft_messages, _) = count_grafts(receivers); assert_eq!( - graft_messages, 12, - "There should be 12 grafts messages sent to peers" + graft_messages, 6, + "There should be 6 grafts messages sent to peers" ); } @@ -668,7 +686,7 @@ fn test_publish_without_flood_publishing() { .unwrap(); let publish_topic = String::from("test_publish"); - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![publish_topic.clone()]) .to_subscribe(true) @@ -695,11 +713,12 @@ fn test_publish_without_flood_publishing() { gs.publish(Topic::new(publish_topic), publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_values() .fold(vec![], |mut collected_publish, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push(message); } } @@ -747,7 +766,7 @@ fn test_fanout() { .unwrap(); let fanout_topic = String::from("test_fanout"); - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![fanout_topic.clone()]) .to_subscribe(true) @@ -779,11 +798,12 @@ fn test_fanout() { ); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_values() .fold(vec![], |mut collected_publish, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push(message); } } @@ -815,10 +835,10 @@ fn test_fanout() { ); } -#[test] /// Test the gossipsub NetworkBehaviour peer connection logic. +#[test] fn test_inject_connected() { - let (gs, peers, queues, topic_hashes) = inject_nodes1() + let (gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -826,11 +846,12 @@ fn test_inject_connected() { // check that our subscriptions are sent to each of the peers // collect all the SendEvents - let subscriptions = queues.into_iter().fold( + let subscriptions = receivers.into_iter().fold( HashMap::>::new(), |mut collected_subscriptions, (peer, c)| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Subscribe(topic)) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Subscribe(topic)) = priority.try_recv() { let mut peer_subs = collected_subscriptions.remove(&peer).unwrap_or_default(); peer_subs.push(topic.into_string()); collected_subscriptions.insert(peer, peer_subs); @@ -860,8 +881,8 @@ fn test_inject_connected() { } } -#[test] /// Test subscription handling +#[test] fn test_handle_received_subscriptions() { // For every subscription: // SUBSCRIBE: - Add subscribed topic to peer_topics for peer. @@ -972,8 +993,8 @@ fn test_handle_received_subscriptions() { ); } -#[test] /// Test Gossipsub.get_random_peers() function +#[test] fn test_get_random_peers() { // generate a default Config let gs_config = ConfigBuilder::default() @@ -1031,7 +1052,7 @@ fn test_get_random_peers() { /// Tests that the correct message is sent when a peer asks for a message in our cache. #[test] fn test_handle_iwant_msg_cached() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(20) .topics(Vec::new()) .to_subscribe(true) @@ -1059,11 +1080,12 @@ fn test_handle_iwant_msg_cached() { gs.handle_iwant(&peers[7], vec![msg_id.clone()]); // the messages we are sending - let sent_messages = queues + let sent_messages = receivers .into_values() .fold(vec![], |mut collected_messages, c| { - while !c.non_priority.is_empty() { - if let Ok(RpcOut::Forward { message, .. }) = c.non_priority.try_recv() { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::Forward { message, .. }) = non_priority.try_recv() { collected_messages.push(message) } } @@ -1082,7 +1104,7 @@ fn test_handle_iwant_msg_cached() { /// Tests that messages are sent correctly depending on the shifting of the message cache. #[test] fn test_handle_iwant_msg_cached_shifted() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, mut receivers, _) = inject_nodes1() .peer_no(20) .topics(Vec::new()) .to_subscribe(true) @@ -1115,21 +1137,29 @@ fn test_handle_iwant_msg_cached_shifted() { gs.handle_iwant(&peers[7], vec![msg_id.clone()]); // is the message is being sent? - let message_exists = queues.values().any(|c| { - let mut out = false; - while !c.non_priority.is_empty() { - if matches!(c.non_priority.try_recv(), Ok(RpcOut::Forward{message, timeout: _ }) if + let mut message_exists = false; + receivers = receivers.into_iter().map(|(peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if matches!(non_priority.try_recv(), Ok(RpcOut::Forward{message, timeout: _ }) if gs.config.message_id( &gs.data_transform .inbound_transform(message.clone()) .unwrap(), ) == msg_id) { - out = true; + message_exists = true; } } - out - }); + ( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: c.priority, + non_priority: non_priority.peekable(), + }, + ) + }).collect(); // default history_length is 5, expect no messages after shift > 5 if shift < 5 { assert!( @@ -1145,8 +1175,8 @@ fn test_handle_iwant_msg_cached_shifted() { } } +/// tests that an event is not created when a peers asks for a message not in our cache #[test] -// tests that an event is not created when a peers asks for a message not in our cache fn test_handle_iwant_msg_not_cached() { let (mut gs, peers, _, _) = inject_nodes1() .peer_no(20) @@ -1164,10 +1194,10 @@ fn test_handle_iwant_msg_not_cached() { ); } +/// tests that an event is created when a peer shares that it has a message we want #[test] -// tests that an event is created when a peer shares that it has a message we want fn test_handle_ihave_subscribed_and_msg_not_cached() { - let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() + let (mut gs, peers, mut receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![String::from("topic1")]) .to_subscribe(true) @@ -1180,9 +1210,10 @@ fn test_handle_ihave_subscribed_and_msg_not_cached() { // check that we sent an IWANT request for `unknown id` let mut iwant_exists = false; - let receiver = receivers.get(&peers[7]).unwrap(); - while !receiver.non_priority.is_empty() { - if let Ok(RpcOut::IWant(IWant { message_ids })) = receiver.non_priority.try_recv() { + let receiver = receivers.remove(&peers[7]).unwrap(); + let non_priority = receiver.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::IWant(IWant { message_ids })) = non_priority.try_recv() { if message_ids .iter() .any(|m| *m == MessageId::new(b"unknown id")) @@ -1199,9 +1230,9 @@ fn test_handle_ihave_subscribed_and_msg_not_cached() { ); } +/// tests that an event is not created when a peer shares that it has a message that +/// we already have #[test] -// tests that an event is not created when a peer shares that it has a message that -// we already have fn test_handle_ihave_subscribed_and_msg_cached() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1221,9 +1252,9 @@ fn test_handle_ihave_subscribed_and_msg_cached() { ) } +/// test that an event is not created when a peer shares that it has a message in +/// a topic that we are not subscribed to #[test] -// test that an event is not created when a peer shares that it has a message in -// a topic that we are not subscribed to fn test_handle_ihave_not_subscribed() { let (mut gs, peers, _, _) = inject_nodes1() .peer_no(20) @@ -1247,9 +1278,9 @@ fn test_handle_ihave_not_subscribed() { ) } +/// tests that a peer is added to our mesh when we are both subscribed +/// to the same topic #[test] -// tests that a peer is added to our mesh when we are both subscribed -// to the same topic fn test_handle_graft_is_subscribed() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1265,9 +1296,9 @@ fn test_handle_graft_is_subscribed() { ); } +/// tests that a peer is not added to our mesh when they are subscribed to +/// a topic that we are not #[test] -// tests that a peer is not added to our mesh when they are subscribed to -// a topic that we are not fn test_handle_graft_is_not_subscribed() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1286,8 +1317,8 @@ fn test_handle_graft_is_not_subscribed() { ); } +/// tests multiple topics in a single graft message #[test] -// tests multiple topics in a single graft message fn test_handle_graft_multiple_topics() { let topics: Vec = ["topic1", "topic2", "topic3", "topic4"] .iter() @@ -1321,8 +1352,8 @@ fn test_handle_graft_multiple_topics() { ); } +/// tests that a peer is removed from our mesh #[test] -// tests that a peer is removed from our mesh fn test_handle_prune_peer_in_mesh() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1352,43 +1383,65 @@ fn test_handle_prune_peer_in_mesh() { } fn count_control_msgs( - queues: &HashMap, + receivers: HashMap, mut filter: impl FnMut(&PeerId, &RpcOut) -> bool, -) -> usize { - queues - .iter() - .fold(0, |mut collected_messages, (peer_id, c)| { - while !c.priority.is_empty() || !c.non_priority.is_empty() { - if let Ok(rpc) = c.priority.try_recv() { - if filter(peer_id, &rpc) { - collected_messages += 1; - } +) -> (usize, HashMap) { + let mut new_receivers = HashMap::new(); + let mut collected_messages = 0; + for (peer_id, c) in receivers.into_iter() { + let priority = c.priority.into_inner(); + let non_priority = c.non_priority.into_inner(); + while !priority.is_empty() || !non_priority.is_empty() { + if let Ok(rpc) = priority.try_recv() { + if filter(&peer_id, &rpc) { + collected_messages += 1; } - if let Ok(rpc) = c.non_priority.try_recv() { - if filter(peer_id, &rpc) { - collected_messages += 1; - } + } + if let Ok(rpc) = non_priority.try_recv() { + if filter(&peer_id, &rpc) { + collected_messages += 1; } } - collected_messages - }) + } + new_receivers.insert( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: priority.peekable(), + non_priority: non_priority.peekable(), + }, + ); + } + (collected_messages, new_receivers) } fn flush_events( gs: &mut Behaviour, - receiver_queues: &HashMap, -) { + receivers: HashMap, +) -> HashMap { gs.events.clear(); - for c in receiver_queues.values() { - while !c.priority.is_empty() || !c.non_priority.is_empty() { - let _ = c.priority.try_recv(); - let _ = c.non_priority.try_recv(); + let mut new_receivers = HashMap::new(); + for (peer_id, c) in receivers.into_iter() { + let priority = c.priority.into_inner(); + let non_priority = c.non_priority.into_inner(); + while !priority.is_empty() || !non_priority.is_empty() { + let _ = priority.try_recv(); + let _ = non_priority.try_recv(); } + new_receivers.insert( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: priority.peekable(), + non_priority: non_priority.peekable(), + }, + ); } + new_receivers } +/// tests that a peer added as explicit peer gets connected to #[test] -// tests that a peer added as explicit peer gets connected to fn test_explicit_peer_gets_connected() { let (mut gs, _, _, _) = inject_nodes1() .peer_no(0) @@ -1423,7 +1476,7 @@ fn test_explicit_peer_reconnects() { .check_explicit_peers_ticks(2) .build() .unwrap(); - let (mut gs, others, queues, _) = inject_nodes1() + let (mut gs, others, receivers, _) = inject_nodes1() .peer_no(1) .topics(Vec::new()) .to_subscribe(true) @@ -1435,7 +1488,7 @@ fn test_explicit_peer_reconnects() { //add peer as explicit peer gs.add_explicit_peer(peer); - flush_events(&mut gs, &queues); + flush_events(&mut gs, receivers); //disconnect peer disconnect_peer(&mut gs, peer); @@ -1473,7 +1526,7 @@ fn test_explicit_peer_reconnects() { #[test] fn test_handle_graft_explicit_peer() { - let (mut gs, peers, queues, topic_hashes) = inject_nodes1() + let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(1) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -1490,21 +1543,24 @@ fn test_handle_graft_explicit_peer() { assert!(gs.mesh[&topic_hashes[1]].is_empty()); //check prunes - assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == peer && match m { - RpcOut::Prune(Prune { topic_hash, .. }) => - topic_hash == &topic_hashes[0] || topic_hash == &topic_hashes[1], + RpcOut::Prune(Prune { topic_hash, .. }) => { + topic_hash == &topic_hashes[0] || topic_hash == &topic_hashes[1] + } _ => false, - }) - >= 2, + } + }); + assert!( + control_msgs >= 2, "Not enough prunes sent when grafting from explicit peer" ); } #[test] fn explicit_peers_not_added_to_mesh_on_receiving_subscription() { - let (gs, peers, queues, topic_hashes) = inject_nodes1() + let (gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(2) .topics(vec![String::from("topic1")]) .to_subscribe(true) @@ -1519,25 +1575,27 @@ fn explicit_peers_not_added_to_mesh_on_receiving_subscription() { ); //assert that graft gets created to non-explicit peer + let (control_msgs, receivers) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && matches!(m, RpcOut::Graft { .. }) + }); assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] - && matches!(m, RpcOut::Graft { .. })) - >= 1, + control_msgs >= 1, "No graft message got created to non-explicit peer" ); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn do_not_graft_explicit_peer() { - let (mut gs, others, queues, topic_hashes) = inject_nodes1() + let (mut gs, others, receivers, topic_hashes) = inject_nodes1() .peer_no(1) .topics(vec![String::from("topic")]) .to_subscribe(true) @@ -1551,17 +1609,18 @@ fn do_not_graft_explicit_peer() { assert_eq!(gs.mesh[&topic_hashes[0]], BTreeSet::new()); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &others[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &others[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn do_forward_messages_to_explicit_peers() { - let (mut gs, peers, queues, topic_hashes) = inject_nodes1() + let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(2) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -1582,9 +1641,10 @@ fn do_forward_messages_to_explicit_peers() { }; gs.handle_received_message(message.clone(), &local_id); assert_eq!( - queues.into_iter().fold(0, |mut fwds, (peer_id, c)| { - while !c.non_priority.is_empty() { - if matches!(c.non_priority.try_recv(), Ok(RpcOut::Forward{message: m, timeout: _}) if peer_id == peers[0] && m.data == message.data) { + receivers.into_iter().fold(0, |mut fwds, (peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if matches!(non_priority.try_recv(), Ok(RpcOut::Forward{message: m, timeout: _}) if peer_id == peers[0] && m.data == message.data) { fwds +=1; } } @@ -1597,7 +1657,7 @@ fn do_forward_messages_to_explicit_peers() { #[test] fn explicit_peers_not_added_to_mesh_on_subscribe() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(2) .topics(Vec::new()) .to_subscribe(true) @@ -1625,25 +1685,27 @@ fn explicit_peers_not_added_to_mesh_on_subscribe() { assert_eq!(gs.mesh[&topic_hash], vec![peers[1]].into_iter().collect()); //assert that graft gets created to non-explicit peer + let (control_msgs, receivers) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && matches!(m, RpcOut::Graft { .. }) + }); assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] - && matches!(m, RpcOut::Graft { .. })) - > 0, + control_msgs > 0, "No graft message got created to non-explicit peer" ); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn explicit_peers_not_added_to_mesh_from_fanout_on_subscribe() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(2) .topics(Vec::new()) .to_subscribe(true) @@ -1674,25 +1736,27 @@ fn explicit_peers_not_added_to_mesh_from_fanout_on_subscribe() { assert_eq!(gs.mesh[&topic_hash], vec![peers[1]].into_iter().collect()); //assert that graft gets created to non-explicit peer + let (control_msgs, receivers) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && matches!(m, RpcOut::Graft { .. }) + }); assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] - && matches!(m, RpcOut::Graft { .. })) - >= 1, + control_msgs >= 1, "No graft message got created to non-explicit peer" ); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn no_gossip_gets_sent_to_explicit_peers() { - let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() + let (mut gs, peers, mut receivers, topic_hashes) = inject_nodes1() .peer_no(2) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -1721,23 +1785,24 @@ fn no_gossip_gets_sent_to_explicit_peers() { } //assert that no gossip gets sent to explicit peer - let receiver = receivers.get(&peers[0]).unwrap(); + let receiver = receivers.remove(&peers[0]).unwrap(); let mut gossips = 0; - while !receiver.non_priority.is_empty() { - if let Ok(RpcOut::IHave(_)) = receiver.non_priority.try_recv() { + let non_priority = receiver.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::IHave(_)) = non_priority.try_recv() { gossips += 1; } } assert_eq!(gossips, 0, "Gossip got emitted to explicit peer"); } -// Tests the mesh maintenance addition +/// Tests the mesh maintenance addition #[test] fn test_mesh_addition() { let config: Config = Config::default(); // Adds mesh_low peers and PRUNE 2 giving us a deficit. - let (mut gs, peers, _queues, topics) = inject_nodes1() + let (mut gs, peers, _receivers, topics) = inject_nodes1() .peer_no(config.mesh_n() + 1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1765,7 +1830,7 @@ fn test_mesh_addition() { assert_eq!(gs.mesh.get(&topics[0]).unwrap().len(), config.mesh_n()); } -// Tests the mesh maintenance subtraction +/// Tests the mesh maintenance subtraction #[test] fn test_mesh_subtraction() { let config = Config::default(); @@ -1853,7 +1918,7 @@ fn test_send_px_and_backoff_in_prune() { let config: Config = Config::default(); //build mesh with enough peers for px - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(config.prune_peers() + 1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1869,24 +1934,25 @@ fn test_send_px_and_backoff_in_prune() { ); //check prune message - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && match m { RpcOut::Prune(Prune { topic_hash, peers, backoff, - }) => + }) => { topic_hash == &topics[0] && peers.len() == config.prune_peers() && //all peers are different peers.iter().collect::>().len() == config.prune_peers() && - backoff.unwrap() == config.prune_backoff().as_secs(), + backoff.unwrap() == config.prune_backoff().as_secs() + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] @@ -1894,7 +1960,7 @@ fn test_prune_backoffed_peer_on_graft() { let config: Config = Config::default(); //build mesh with enough peers for px - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(config.prune_peers() + 1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1911,28 +1977,29 @@ fn test_prune_backoffed_peer_on_graft() { ); //ignore all messages until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //handle graft gs.handle_graft(&peers[0], vec![topics[0].clone()]); //check prune message - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && match m { RpcOut::Prune(Prune { topic_hash, peers, backoff, - }) => + }) => { topic_hash == &topics[0] && //no px in this case peers.is_empty() && - backoff.unwrap() == config.prune_backoff().as_secs(), + backoff.unwrap() == config.prune_backoff().as_secs() + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] @@ -1943,7 +2010,7 @@ fn test_do_not_graft_within_backoff_period() { .build() .unwrap(); //only one peer => mesh too small and will try to regraft as early as possible - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1954,7 +2021,7 @@ fn test_do_not_graft_within_backoff_period() { gs.handle_prune(&peers[0], vec![(topics[0].clone(), Vec::new(), Some(1))]); //forget all events until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //call heartbeat gs.heartbeat(); @@ -1967,9 +2034,10 @@ fn test_do_not_graft_within_backoff_period() { //Check that no graft got created (we have backoff_slack = 1 therefore one more heartbeat // is needed). + let (control_msgs, receivers) = + count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert_eq!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "Graft message created too early within backoff period" ); @@ -1978,8 +2046,9 @@ fn test_do_not_graft_within_backoff_period() { gs.heartbeat(); //check that graft got created + let (control_msgs, _) = count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })) > 0, + control_msgs > 0, "No graft message was created after backoff period" ); } @@ -1994,7 +2063,7 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without .build() .unwrap(); //only one peer => mesh too small and will try to regraft as early as possible - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2005,7 +2074,7 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without gs.handle_prune(&peers[0], vec![(topics[0].clone(), Vec::new(), None)]); //forget all events until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //call heartbeat gs.heartbeat(); @@ -2016,9 +2085,10 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without //Check that no graft got created (we have backoff_slack = 1 therefore one more heartbeat // is needed). + let (control_msgs, receivers) = + count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert_eq!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "Graft message created too early within backoff period" ); @@ -2027,8 +2097,9 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without gs.heartbeat(); //check that graft got created + let (control_msgs, _) = count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })) > 0, + control_msgs > 0, "No graft message was created after backoff period" ); } @@ -2047,7 +2118,7 @@ fn test_unsubscribe_backoff() { let topic = String::from("test"); // only one peer => mesh too small and will try to regraft as early as possible - let (mut gs, _, queues, topics) = inject_nodes1() + let (mut gs, _, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec![topic.clone()]) .to_subscribe(true) @@ -2056,19 +2127,19 @@ fn test_unsubscribe_backoff() { let _ = gs.unsubscribe(&Topic::new(topic)); + let (control_msgs, receivers) = count_control_msgs(receivers, |_, m| match m { + RpcOut::Prune(Prune { backoff, .. }) => backoff == &Some(1), + _ => false, + }); assert_eq!( - count_control_msgs(&queues, |_, m| match m { - RpcOut::Prune(Prune { backoff, .. }) => backoff == &Some(1), - _ => false, - }), - 1, + control_msgs, 1, "Peer should be pruned with `unsubscribe_backoff`." ); let _ = gs.subscribe(&Topic::new(topics[0].to_string())); // forget all events until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); // call heartbeat gs.heartbeat(); @@ -2081,9 +2152,10 @@ fn test_unsubscribe_backoff() { // Check that no graft got created (we have backoff_slack = 1 therefore one more heartbeat // is needed). + let (control_msgs, receivers) = + count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert_eq!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "Graft message created too early within backoff period" ); @@ -2092,8 +2164,9 @@ fn test_unsubscribe_backoff() { gs.heartbeat(); // check that graft got created + let (control_msgs, _) = count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })) > 0, + control_msgs > 0, "No graft message was created after backoff period" ); } @@ -2104,7 +2177,7 @@ fn test_flood_publish() { let topic = "test"; // Adds more peers than mesh can hold to test flood publishing - let (mut gs, _, queues, _) = inject_nodes1() + let (mut gs, _, receivers, _) = inject_nodes1() .peer_no(config.mesh_n_high() + 10) .topics(vec![topic.into()]) .to_subscribe(true) @@ -2115,11 +2188,12 @@ fn test_flood_publish() { gs.publish(Topic::new(topic), publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_values() .fold(vec![], |mut collected_publish, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push(message); } } @@ -2158,7 +2232,7 @@ fn test_gossip_to_at_least_gossip_lazy_peers() { //add more peers than in mesh to test gossipping //by default only mesh_n_low peers will get added to mesh - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(config.mesh_n_low() + config.gossip_lazy() + 1) .topics(vec!["topic".into()]) .to_subscribe(true) @@ -2185,16 +2259,14 @@ fn test_gossip_to_at_least_gossip_lazy_peers() { let msg_id = gs.config.message_id(message); //check that exactly config.gossip_lazy() many gossip messages were sent. - assert_eq!( - count_control_msgs(&queues, |_, action| match action { - RpcOut::IHave(IHave { - topic_hash, - message_ids, - }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), - _ => false, - }), - config.gossip_lazy() - ); + let (control_msgs, _) = count_control_msgs(receivers, |_, action| match action { + RpcOut::IHave(IHave { + topic_hash, + message_ids, + }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), + _ => false, + }); + assert_eq!(control_msgs, config.gossip_lazy()); } #[test] @@ -2203,7 +2275,7 @@ fn test_gossip_to_at_most_gossip_factor_peers() { //add a lot of peers let m = config.mesh_n_low() + config.gossip_lazy() * (2.0 / config.gossip_factor()) as usize; - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(m) .topics(vec!["topic".into()]) .to_subscribe(true) @@ -2229,14 +2301,15 @@ fn test_gossip_to_at_most_gossip_factor_peers() { let msg_id = gs.config.message_id(message); //check that exactly config.gossip_lazy() many gossip messages were sent. + let (control_msgs, _) = count_control_msgs(receivers, |_, action| match action { + RpcOut::IHave(IHave { + topic_hash, + message_ids, + }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), + _ => false, + }); assert_eq!( - count_control_msgs(&queues, |_, action| match action { - RpcOut::IHave(IHave { - topic_hash, - message_ids, - }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), - _ => false, - }), + control_msgs, ((m - config.mesh_n_low()) as f64 * config.gossip_factor()) as usize ); } @@ -2365,7 +2438,7 @@ fn test_prune_negative_scored_peers() { let config = Config::default(); //build mesh with one peer - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2388,22 +2461,23 @@ fn test_prune_negative_scored_peers() { assert!(gs.mesh[&topics[0]].is_empty()); //check prune message - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && match m { RpcOut::Prune(Prune { topic_hash, peers, backoff, - }) => + }) => { topic_hash == &topics[0] && //no px in this case peers.is_empty() && - backoff.unwrap() == config.prune_backoff().as_secs(), + backoff.unwrap() == config.prune_backoff().as_secs() + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] @@ -2496,7 +2570,7 @@ fn test_only_send_nonnegative_scoring_peers_in_px() { .unwrap(); // Build mesh with three peer - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(3) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2522,29 +2596,26 @@ fn test_only_send_nonnegative_scoring_peers_in_px() { ); // Check that px in prune message only contains third peer - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && match m { RpcOut::Prune(Prune { topic_hash, peers: px, .. - }) => + }) => { topic_hash == &topics[0] && px.len() == 1 - && px[0].peer_id.as_ref().unwrap() == &peers[2], + && px[0].peer_id.as_ref().unwrap() == &peers[2] + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] fn test_do_not_gossip_to_peers_below_gossip_threshold() { - // use tracing_subscriber::EnvFilter; - // let _ = tracing_subscriber::fmt() - // .with_env_filter(EnvFilter::from_default_env()) - // .try_init(); let config = Config::default(); let peer_score_params = PeerScoreParams::default(); let peer_score_thresholds = PeerScoreThresholds { @@ -2601,23 +2672,21 @@ fn test_do_not_gossip_to_peers_below_gossip_threshold() { gs.emit_gossip(); // Check that exactly one gossip messages got sent and it got sent to p2 - assert_eq!( - count_control_msgs(&receivers, |peer, action| match action { - RpcOut::IHave(IHave { - topic_hash, - message_ids, - }) => { - if topic_hash == &topics[0] && message_ids.iter().any(|id| id == &msg_id) { - assert_eq!(peer, &p2); - true - } else { - false - } + let (control_msgs, _) = count_control_msgs(receivers, |peer, action| match action { + RpcOut::IHave(IHave { + topic_hash, + message_ids, + }) => { + if topic_hash == &topics[0] && message_ids.iter().any(|id| id == &msg_id) { + assert_eq!(peer, &p2); + true + } else { + false } - _ => false, - }), - 1 - ); + } + _ => false, + }); + assert_eq!(control_msgs, 1); } #[test] @@ -2630,7 +2699,7 @@ fn test_iwant_msg_from_peer_below_gossip_threshold_gets_ignored() { }; // Build full mesh - let (mut gs, peers, mut queues, topics) = inject_nodes1() + let (mut gs, peers, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2647,9 +2716,9 @@ fn test_iwant_msg_from_peer_below_gossip_threshold_gets_ignored() { // Add two additional peers that will not be part of the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); // Reduce score of p1 below peer_score_thresholds.gossip_threshold // note that penalties get squared so two penalties means a score of @@ -2680,16 +2749,18 @@ fn test_iwant_msg_from_peer_below_gossip_threshold_gets_ignored() { gs.handle_iwant(&p2, vec![msg_id.clone()]); // the messages we are sending - let sent_messages = queues - .into_iter() - .fold(vec![], |mut collected_messages, (peer_id, c)| { - while !c.non_priority.is_empty() { - if let Ok(RpcOut::Forward { message, .. }) = c.non_priority.try_recv() { - collected_messages.push((peer_id, message)); + let sent_messages = + receivers + .into_iter() + .fold(vec![], |mut collected_messages, (peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::Forward { message, .. }) = non_priority.try_recv() { + collected_messages.push((peer_id, message)); + } } - } - collected_messages - }); + collected_messages + }); //the message got sent to p2 assert!(sent_messages @@ -2718,7 +2789,7 @@ fn test_ihave_msg_from_peer_below_gossip_threshold_gets_ignored() { ..PeerScoreThresholds::default() }; //build full mesh - let (mut gs, peers, mut queues, topics) = inject_nodes1() + let (mut gs, peers, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2735,9 +2806,9 @@ fn test_ihave_msg_from_peer_below_gossip_threshold_gets_ignored() { //add two additional peers that will not be part of the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //reduce score of p1 below peer_score_thresholds.gossip_threshold //note that penalties get squared so two penalties means a score of @@ -2767,19 +2838,18 @@ fn test_ihave_msg_from_peer_below_gossip_threshold_gets_ignored() { gs.handle_ihave(&p2, vec![(topics[0].clone(), vec![msg_id.clone()])]); // check that we sent exactly one IWANT request to p2 - assert_eq!( - count_control_msgs(&queues, |peer, c| match c { - RpcOut::IWant(IWant { message_ids }) => - if message_ids.iter().any(|m| m == &msg_id) { - assert_eq!(peer, &p2); - true - } else { - false - }, - _ => false, - }), - 1 - ); + let (control_msgs, _) = count_control_msgs(receivers, |peer, c| match c { + RpcOut::IWant(IWant { message_ids }) => { + if message_ids.iter().any(|m| m == &msg_id) { + assert_eq!(peer, &p2); + true + } else { + false + } + } + _ => false, + }); + assert_eq!(control_msgs, 1); } #[test] @@ -2796,7 +2866,7 @@ fn test_do_not_publish_to_peer_below_publish_threshold() { }; //build mesh with no peers and no subscribed topics - let (mut gs, _, mut queues, _) = inject_nodes1() + let (mut gs, _, mut receivers, _) = inject_nodes1() .gs_config(config) .scoring(Some((peer_score_params, peer_score_thresholds))) .create_network(); @@ -2807,9 +2877,9 @@ fn test_do_not_publish_to_peer_below_publish_threshold() { //add two additional peers that will be added to the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //reduce score of p1 below peer_score_thresholds.publish_threshold //note that penalties get squared so two penalties means a score of @@ -2827,11 +2897,12 @@ fn test_do_not_publish_to_peer_below_publish_threshold() { gs.publish(topic, publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_iter() .fold(vec![], |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push((peer_id, message)); } } @@ -2853,7 +2924,7 @@ fn test_do_not_flood_publish_to_peer_below_publish_threshold() { ..PeerScoreThresholds::default() }; //build mesh with no peers - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .topics(vec!["test".into()]) .gs_config(config) .scoring(Some((peer_score_params, peer_score_thresholds))) @@ -2861,9 +2932,9 @@ fn test_do_not_flood_publish_to_peer_below_publish_threshold() { //add two additional peers that will be added to the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //reduce score of p1 below peer_score_thresholds.publish_threshold //note that penalties get squared so two penalties means a score of @@ -2881,11 +2952,12 @@ fn test_do_not_flood_publish_to_peer_below_publish_threshold() { gs.publish(Topic::new("test"), publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_iter() .fold(vec![], |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push((peer_id, message)) } } @@ -4351,7 +4423,7 @@ fn test_opportunistic_grafting() { #[test] fn test_ignore_graft_from_unknown_topic() { //build gossipsub without subscribing to any topics - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(1) .topics(vec![]) .to_subscribe(false) @@ -4361,9 +4433,9 @@ fn test_ignore_graft_from_unknown_topic() { gs.handle_graft(&peers[0], vec![Topic::new("test").hash()]); //assert that no prune got created + let (control_msgs, _) = count_control_msgs(receivers, |_, a| matches!(a, RpcOut::Prune { .. })); assert_eq!( - count_control_msgs(&queues, |_, a| matches!(a, RpcOut::Prune { .. })), - 0, + control_msgs, 0, "we should not prune after graft in unknown topic" ); } @@ -4372,7 +4444,7 @@ fn test_ignore_graft_from_unknown_topic() { fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { let config = Config::default(); //build gossipsub with full mesh - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4380,7 +4452,7 @@ fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { //add another peer not in the mesh let (peer, receiver) = add_peer(&mut gs, &topics, false, false); - queues.insert(peer, receiver); + receivers.insert(peer, receiver); //receive a message let mut seq = 0; @@ -4394,7 +4466,7 @@ fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { gs.handle_received_message(m1, &PeerId::random()); //clear events - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //the first gossip_retransimission many iwants return the valid message, all others are // ignored. @@ -4403,9 +4475,10 @@ fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { } assert_eq!( - queues.into_values().fold(0, |mut fwds, c| { - while !c.non_priority.is_empty() { - if let Ok(RpcOut::Forward { .. }) = c.non_priority.try_recv() { + receivers.into_values().fold(0, |mut fwds, c| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::Forward { .. }) = non_priority.try_recv() { fwds += 1; } } @@ -4460,10 +4533,12 @@ fn test_ignore_too_many_ihaves() { .collect(); //we send iwant only for the first 10 messages + let (control_msgs, receivers) = count_control_msgs(receivers, |p, action| { + p == &peer + && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1 && first_ten.contains(&message_ids[0])) + }); assert_eq!( - count_control_msgs(&receivers, |p, action| p == &peer - && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1 && first_ten.contains(&message_ids[0]))), - 10, + control_msgs, 10, "exactly the first ten ihaves should be processed and one iwant for each created" ); @@ -4484,12 +4559,11 @@ fn test_ignore_too_many_ihaves() { } //we sent iwant for all 10 messages - assert_eq!( - count_control_msgs(&receivers, |p, action| p == &peer - && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1)), - 10, - "all 20 should get sent" - ); + let (control_msgs, _) = count_control_msgs(receivers, |p, action| { + p == &peer + && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1) + }); + assert_eq!(control_msgs, 10, "all 20 should get sent"); } #[test] @@ -4500,7 +4574,7 @@ fn test_ignore_too_many_messages_in_ihave() { .build() .unwrap(); //build gossipsub with full mesh - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4509,7 +4583,7 @@ fn test_ignore_too_many_messages_in_ihave() { //add another peer not in the mesh let (peer, receiver) = add_peer(&mut gs, &topics, false, false); - queues.insert(peer, receiver); + receivers.insert(peer, receiver); //peer has 20 messages let mut seq = 0; @@ -4534,18 +4608,18 @@ fn test_ignore_too_many_messages_in_ihave() { //we send iwant only for the first 10 messages let mut sum = 0; - assert_eq!( - count_control_msgs(&queues, |p, rpc| match rpc { - RpcOut::IWant(IWant { message_ids }) => { - p == &peer && { - assert!(first_twelve.is_superset(&message_ids.iter().collect())); - sum += message_ids.len(); - true - } + let (control_msgs, receivers) = count_control_msgs(receivers, |p, rpc| match rpc { + RpcOut::IWant(IWant { message_ids }) => { + p == &peer && { + assert!(first_twelve.is_superset(&message_ids.iter().collect())); + sum += message_ids.len(); + true } - _ => false, - }), - 2, + } + _ => false, + }); + assert_eq!( + control_msgs, 2, "the third ihave should get ignored and no iwant sent" ); @@ -4560,20 +4634,16 @@ fn test_ignore_too_many_messages_in_ihave() { //we sent 10 iwant messages ids via a IWANT rpc. let mut sum = 0; - assert_eq!( - count_control_msgs(&queues, |p, rpc| { - match rpc { - RpcOut::IWant(IWant { message_ids }) => { - p == &peer && { - sum += message_ids.len(); - true - } - } - _ => false, + let (control_msgs, _) = count_control_msgs(receivers, |p, rpc| match rpc { + RpcOut::IWant(IWant { message_ids }) => { + p == &peer && { + sum += message_ids.len(); + true } - }), - 1 - ); + } + _ => false, + }); + assert_eq!(control_msgs, 1); assert_eq!(sum, 10, "exactly 20 iwants should get sent"); } @@ -4619,22 +4689,22 @@ fn test_limit_number_of_message_ids_inside_ihave() { let mut ihaves1 = HashSet::new(); let mut ihaves2 = HashSet::new(); - assert_eq!( - count_control_msgs(&receivers, |p, action| match action { - RpcOut::IHave(IHave { message_ids, .. }) => { - if p == &p1 { - ihaves1 = message_ids.iter().cloned().collect(); - true - } else if p == &p2 { - ihaves2 = message_ids.iter().cloned().collect(); - true - } else { - false - } + let (control_msgs, _) = count_control_msgs(receivers, |p, action| match action { + RpcOut::IHave(IHave { message_ids, .. }) => { + if p == &p1 { + ihaves1 = message_ids.iter().cloned().collect(); + true + } else if p == &p2 { + ihaves2 = message_ids.iter().cloned().collect(); + true + } else { + false } - _ => false, - }), - 2, + } + _ => false, + }); + assert_eq!( + control_msgs, 2, "should have emitted one ihave to p1 and one to p2" ); @@ -4668,7 +4738,6 @@ fn test_iwant_penalties() { .with_env_filter(EnvFilter::from_default_env()) .try_init(); */ - let config = ConfigBuilder::default() .iwant_followup_time(Duration::from_secs(4)) .build() @@ -4791,7 +4860,7 @@ fn test_publish_to_floodsub_peers_without_flood_publish() { .flood_publish(false) .build() .unwrap(); - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_low() - 1) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4807,11 +4876,11 @@ fn test_publish_to_floodsub_peers_without_flood_publish() { Multiaddr::empty(), Some(PeerKind::Floodsub), ); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer_with_addr_and_kind(&mut gs, &topics, false, false, Multiaddr::empty(), None); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //p1 and p2 are not in the mesh assert!(!gs.mesh[&topics[0]].contains(&p1) && !gs.mesh[&topics[0]].contains(&p2)); @@ -4821,12 +4890,13 @@ fn test_publish_to_floodsub_peers_without_flood_publish() { gs.publish(Topic::new("test"), publish_data).unwrap(); // Collect publish messages to floodsub peers - let publishes = queues - .iter() + let publishes = receivers + .into_iter() .fold(0, |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if matches!(c.priority.try_recv(), - Ok(RpcOut::Publish{..}) if peer_id == &p1 || peer_id == &p2) + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if matches!(priority.try_recv(), + Ok(RpcOut::Publish{..}) if peer_id == p1 || peer_id == p2) { collected_publish += 1; } @@ -4846,7 +4916,7 @@ fn test_do_not_use_floodsub_in_fanout() { .flood_publish(false) .build() .unwrap(); - let (mut gs, _, mut queues, _) = inject_nodes1() + let (mut gs, _, mut receivers, _) = inject_nodes1() .peer_no(config.mesh_n_low() - 1) .topics(Vec::new()) .to_subscribe(false) @@ -4866,22 +4936,23 @@ fn test_do_not_use_floodsub_in_fanout() { Some(PeerKind::Floodsub), ); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer_with_addr_and_kind(&mut gs, &topics, false, false, Multiaddr::empty(), None); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //publish a message let publish_data = vec![0; 42]; gs.publish(Topic::new("test"), publish_data).unwrap(); // Collect publish messages to floodsub peers - let publishes = queues - .iter() + let publishes = receivers + .into_iter() .fold(0, |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if matches!(c.priority.try_recv(), - Ok(RpcOut::Publish{..}) if peer_id == &p1 || peer_id == &p2) + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if matches!(priority.try_recv(), + Ok(RpcOut::Publish{..}) if peer_id == p1 || peer_id == p2) { collected_publish += 1; } @@ -4932,7 +5003,7 @@ fn test_dont_add_floodsub_peers_to_mesh_on_join() { #[test] fn test_dont_send_px_to_old_gossipsub_peers() { - let (mut gs, _, queues, topics) = inject_nodes1() + let (mut gs, _, receivers, topics) = inject_nodes1() .peer_no(0) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4956,20 +5027,17 @@ fn test_dont_send_px_to_old_gossipsub_peers() { ); //check that prune does not contain px - assert_eq!( - count_control_msgs(&queues, |_, m| match m { - RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), - _ => false, - }), - 0, - "Should not send px to floodsub peers" - ); + let (control_msgs, _) = count_control_msgs(receivers, |_, m| match m { + RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), + _ => false, + }); + assert_eq!(control_msgs, 0, "Should not send px to floodsub peers"); } #[test] fn test_dont_send_floodsub_peers_in_px() { //build mesh with one peer - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -4994,14 +5062,11 @@ fn test_dont_send_floodsub_peers_in_px() { ); //check that px in prune message is empty - assert_eq!( - count_control_msgs(&queues, |_, m| match m { - RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), - _ => false, - }), - 0, - "Should not include floodsub peers in px" - ); + let (control_msgs, _) = count_control_msgs(receivers, |_, m| match m { + RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), + _ => false, + }); + assert_eq!(control_msgs, 0, "Should not include floodsub peers in px"); } #[test] @@ -5088,7 +5153,7 @@ fn test_subscribe_and_graft_with_negative_score() { ))) .create_network(); - let (mut gs2, _, queues, _) = inject_nodes1().create_network(); + let (mut gs2, _, receivers, _) = inject_nodes1().create_network(); let connection_id = ConnectionId::new_unchecked(0); @@ -5105,37 +5170,41 @@ fn test_subscribe_and_graft_with_negative_score() { //subscribe to topic in gs2 gs2.subscribe(&topic).unwrap(); - let forward_messages_to_p1 = |gs1: &mut Behaviour<_, _>, _gs2: &mut Behaviour<_, _>| { - //collect messages to p1 - let messages_to_p1 = - queues - .iter() - .filter_map(|(peer_id, c)| match c.non_priority.try_recv() { - Ok(rpc) if peer_id == &p1 => Some(rpc), - _ => None, - }); - - for message in messages_to_p1 { - gs1.on_connection_handler_event( - p2, - connection_id, - HandlerEvent::Message { - rpc: proto_to_message(&message.into_protobuf()), - invalid_messages: vec![], - }, - ); + let forward_messages_to_p1 = |gs1: &mut Behaviour<_, _>, + p1: PeerId, + p2: PeerId, + connection_id: ConnectionId, + receivers: HashMap| + -> HashMap { + let new_receivers = HashMap::new(); + for (peer_id, receiver) in receivers.into_iter() { + let non_priority = receiver.non_priority.into_inner(); + match non_priority.try_recv() { + Ok(rpc) if peer_id == p1 => { + gs1.on_connection_handler_event( + p2, + connection_id, + HandlerEvent::Message { + rpc: proto_to_message(&rpc.into_protobuf()), + invalid_messages: vec![], + }, + ); + } + _ => {} + } } + new_receivers }; //forward the subscribe message - forward_messages_to_p1(&mut gs1, &mut gs2); + let receivers = forward_messages_to_p1(&mut gs1, p1, p2, connection_id, receivers); //heartbeats on both gs1.heartbeat(); gs2.heartbeat(); //forward messages again - forward_messages_to_p1(&mut gs1, &mut gs2); + forward_messages_to_p1(&mut gs1, p1, p2, connection_id, receivers); //nobody got penalized assert!(gs1.peer_score.as_ref().unwrap().0.score(&p2) >= original_score); diff --git a/beacon_node/lighthouse_network/src/gossipsub/handler.rs b/beacon_node/lighthouse_network/src/gossipsub/handler.rs index a8a980ae87e..298570955fc 100644 --- a/beacon_node/lighthouse_network/src/gossipsub/handler.rs +++ b/beacon_node/lighthouse_network/src/gossipsub/handler.rs @@ -176,12 +176,6 @@ impl Handler { } impl EnabledHandler { - #[cfg(test)] - /// For testing purposed obtain the RPCReceiver - pub fn receiver(&mut self) -> RpcReceiver { - self.send_queue.clone() - } - fn on_fully_negotiated_inbound( &mut self, (substream, peer_kind): (Framed, PeerKind), @@ -237,7 +231,7 @@ impl EnabledHandler { } // determine if we need to create the outbound stream - if !self.send_queue.is_empty() + if !self.send_queue.poll_is_empty(cx) && self.outbound_substream.is_none() && !self.outbound_substream_establishing { @@ -247,10 +241,6 @@ impl EnabledHandler { }); } - // We may need to inform the behviour if we have a dropped a message. This gets set if that - // is the case. - let mut dropped_message = None; - // process outbound stream loop { match std::mem::replace( @@ -271,10 +261,11 @@ impl EnabledHandler { } => { if Pin::new(timeout).poll(cx).is_ready() { // Inform the behaviour and end the poll. - dropped_message = Some(HandlerEvent::MessageDropped(message)); self.outbound_substream = Some(OutboundSubstreamState::WaitingOutput(substream)); - break; + return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( + HandlerEvent::MessageDropped(message), + )); } } _ => {} // All other messages are not time-bound. @@ -348,13 +339,7 @@ impl EnabledHandler { } } - // If there was a timeout in sending a message, inform the behaviour before restarting the - // poll - if let Some(handler_event) = dropped_message { - return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour(handler_event)); - } - - // Handle inbound messages + // Handle inbound messages. loop { match std::mem::replace( &mut self.inbound_substream, @@ -419,6 +404,13 @@ impl EnabledHandler { } } + // Drop the next message in queue if it's stale. + if let Poll::Ready(Some(rpc)) = self.send_queue.poll_stale(cx) { + return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( + HandlerEvent::MessageDropped(rpc), + )); + } + Poll::Pending } } diff --git a/beacon_node/lighthouse_network/src/gossipsub/types.rs b/beacon_node/lighthouse_network/src/gossipsub/types.rs index b7bcbf6b3ac..f77185c7c58 100644 --- a/beacon_node/lighthouse_network/src/gossipsub/types.rs +++ b/beacon_node/lighthouse_network/src/gossipsub/types.rs @@ -22,7 +22,8 @@ use crate::gossipsub::metrics::Metrics; use crate::gossipsub::TopicHash; use async_channel::{Receiver, Sender}; -use futures::Stream; +use futures::stream::Peekable; +use futures::{Future, Stream, StreamExt}; use futures_timer::Delay; use instant::Duration; use libp2p::identity::PeerId; @@ -33,7 +34,7 @@ use std::collections::BTreeSet; use std::fmt::Debug; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::task::Poll; +use std::task::{Context, Poll}; use std::{fmt, pin::Pin}; use crate::gossipsub::rpc_proto::proto; @@ -591,9 +592,10 @@ impl fmt::Display for PeerKind { pub(crate) struct RpcSender { cap: usize, len: Arc, - priority: Sender, - non_priority: Sender, - receiver: RpcReceiver, + pub(crate) priority_sender: Sender, + pub(crate) non_priority_sender: Sender, + priority_receiver: Receiver, + non_priority_receiver: Receiver, } impl RpcSender { @@ -602,29 +604,29 @@ impl RpcSender { let (priority_sender, priority_receiver) = async_channel::unbounded(); let (non_priority_sender, non_priority_receiver) = async_channel::bounded(cap / 2); let len = Arc::new(AtomicUsize::new(0)); - let receiver = RpcReceiver { - priority_len: len.clone(), - priority: priority_receiver, - non_priority: non_priority_receiver, - }; RpcSender { cap: cap / 2, len, - priority: priority_sender, - non_priority: non_priority_sender, - receiver: receiver.clone(), + priority_sender, + non_priority_sender, + priority_receiver, + non_priority_receiver, } } /// Create a new Receiver to the sender. pub(crate) fn new_receiver(&self) -> RpcReceiver { - self.receiver.clone() + RpcReceiver { + priority_len: self.len.clone(), + priority: self.priority_receiver.clone().peekable(), + non_priority: self.non_priority_receiver.clone().peekable(), + } } /// Send a `RpcOut::Graft` message to the `RpcReceiver` /// this is high priority. pub(crate) fn graft(&mut self, graft: Graft) { - self.priority + self.priority_sender .try_send(RpcOut::Graft(graft)) .expect("Channel is unbounded and should always be open"); } @@ -632,7 +634,7 @@ impl RpcSender { /// Send a `RpcOut::Prune` message to the `RpcReceiver` /// this is high priority. pub(crate) fn prune(&mut self, prune: Prune) { - self.priority + self.priority_sender .try_send(RpcOut::Prune(prune)) .expect("Channel is unbounded and should always be open"); } @@ -641,7 +643,7 @@ impl RpcSender { /// this is low priority, if the queue is full an Err is returned. #[allow(clippy::result_large_err)] pub(crate) fn ihave(&mut self, ihave: IHave) -> Result<(), RpcOut> { - self.non_priority + self.non_priority_sender .try_send(RpcOut::IHave(ihave)) .map_err(|err| err.into_inner()) } @@ -650,7 +652,7 @@ impl RpcSender { /// this is low priority, if the queue is full an Err is returned. #[allow(clippy::result_large_err)] pub(crate) fn iwant(&mut self, iwant: IWant) -> Result<(), RpcOut> { - self.non_priority + self.non_priority_sender .try_send(RpcOut::IWant(iwant)) .map_err(|err| err.into_inner()) } @@ -658,7 +660,7 @@ impl RpcSender { /// Send a `RpcOut::Subscribe` message to the `RpcReceiver` /// this is high priority. pub(crate) fn subscribe(&mut self, topic: TopicHash) { - self.priority + self.priority_sender .try_send(RpcOut::Subscribe(topic)) .expect("Channel is unbounded and should always be open"); } @@ -666,7 +668,7 @@ impl RpcSender { /// Send a `RpcOut::Unsubscribe` message to the `RpcReceiver` /// this is high priority. pub(crate) fn unsubscribe(&mut self, topic: TopicHash) { - self.priority + self.priority_sender .try_send(RpcOut::Unsubscribe(topic)) .expect("Channel is unbounded and should always be open"); } @@ -682,7 +684,7 @@ impl RpcSender { if self.len.load(Ordering::Relaxed) >= self.cap { return Err(()); } - self.priority + self.priority_sender .try_send(RpcOut::Publish { message: message.clone(), timeout: Delay::new(timeout), @@ -705,7 +707,7 @@ impl RpcSender { timeout: Duration, metrics: Option<&mut Metrics>, ) -> Result<(), ()> { - self.non_priority + self.non_priority_sender .try_send(RpcOut::Forward { message: message.clone(), timeout: Delay::new(timeout), @@ -726,25 +728,73 @@ impl RpcSender { /// Returns the current size of the non-priority queue. pub(crate) fn non_priority_len(&self) -> usize { - self.non_priority.len() + self.non_priority_sender.len() } } /// `RpcOut` sender that is priority aware. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct RpcReceiver { /// The maximum length of the priority queue. - priority_len: Arc, + pub(crate) priority_len: Arc, /// The priority queue receiver. - pub(crate) priority: Receiver, + pub(crate) priority: Peekable>, /// The non priority queue receiver. - pub(crate) non_priority: Receiver, + pub(crate) non_priority: Peekable>, } impl RpcReceiver { - /// Check if both queues are empty. - pub(crate) fn is_empty(&self) -> bool { - self.priority.is_empty() && self.non_priority.is_empty() + // Peek the next message in the queues and return it if its timeout has elapsed. + // Returns `None` if there aren't any more messages on the stream or none is stale. + pub(crate) fn poll_stale(&mut self, cx: &mut Context<'_>) -> Poll> { + // Peek priority queue. + let priority = match Pin::new(&mut self.priority).poll_peek_mut(cx) { + Poll::Ready(Some(RpcOut::Publish { + message: _, + ref mut timeout, + })) => { + if Pin::new(timeout).poll(cx).is_ready() { + // Return the message. + let dropped = futures::ready!(self.priority.poll_next_unpin(cx)) + .expect("There should be a message"); + return Poll::Ready(Some(dropped)); + } + Poll::Ready(None) + } + poll => poll, + }; + + let non_priority = match Pin::new(&mut self.non_priority).poll_peek_mut(cx) { + Poll::Ready(Some(RpcOut::Forward { + message: _, + ref mut timeout, + })) => { + if Pin::new(timeout).poll(cx).is_ready() { + // Return the message. + let dropped = futures::ready!(self.non_priority.poll_next_unpin(cx)) + .expect("There should be a message"); + return Poll::Ready(Some(dropped)); + } + Poll::Ready(None) + } + poll => poll, + }; + + match (priority, non_priority) { + (Poll::Ready(None), Poll::Ready(None)) => Poll::Ready(None), + _ => Poll::Pending, + } + } + + /// Poll queues and return true if both are empty. + pub(crate) fn poll_is_empty(&mut self, cx: &mut Context<'_>) -> bool { + matches!( + ( + Pin::new(&mut self.priority).poll_peek(cx), + Pin::new(&mut self.non_priority).poll_peek(cx), + ), + (Poll::Ready(None), Poll::Ready(None)) + ) } } From 7c236251933dc87760d8c1ed052c7dbac67d497b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Feb 2024 15:18:23 +1100 Subject: [PATCH 02/21] Quieten gossip republish logs (#5235) * Quieten gossip republish logs --- .../lighthouse_network/src/service/mod.rs | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 879f13ad7c8..401e43a53ff 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1226,22 +1226,39 @@ impl Network { .publish(Topic::from(topic.clone()), data) { Ok(_) => { - warn!(self.log, "Gossip message published on retry"; "topic" => topic_str); - if let Some(v) = metrics::get_int_counter( + debug!( + self.log, + "Gossip message published on retry"; + "topic" => topic_str + ); + metrics::inc_counter_vec( &metrics::GOSSIP_LATE_PUBLISH_PER_TOPIC_KIND, &[topic_str], - ) { - v.inc() - }; + ); + } + Err(PublishError::Duplicate) => { + debug!( + self.log, + "Gossip message publish ignored on retry"; + "reason" => "duplicate", + "topic" => topic_str + ); + metrics::inc_counter_vec( + &metrics::GOSSIP_FAILED_LATE_PUBLISH_PER_TOPIC_KIND, + &[topic_str], + ); } Err(e) => { - warn!(self.log, "Gossip message publish failed on retry"; "topic" => topic_str, "error" => %e); - if let Some(v) = metrics::get_int_counter( + warn!( + self.log, + "Gossip message publish failed on retry"; + "topic" => topic_str, + "error" => %e + ); + metrics::inc_counter_vec( &metrics::GOSSIP_FAILED_LATE_PUBLISH_PER_TOPIC_KIND, &[topic_str], - ) { - v.inc() - }; + ); } } } From 0e819fa78543ab9d8e4d626ed5df357915535a2e Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Feb 2024 23:23:51 +1100 Subject: [PATCH 03/21] Schedule Deneb on mainnet (#5233) * Schedule Deneb on mainnet * Fix trusted setup roundtrip test * Fix BN CLI tests for insecure genesis sync --- .../mainnet/config.yaml | 68 ++++++++++++++----- common/eth2_network_config/src/lib.rs | 4 +- consensus/types/presets/mainnet/deneb.yaml | 2 +- consensus/types/src/chain_spec.rs | 2 +- lighthouse/tests/beacon_node.rs | 35 ++++++++-- 5 files changed, 84 insertions(+), 27 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index 019eda43243..b29ecfc6d38 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -1,9 +1,15 @@ # Mainnet config # Extends the mainnet preset -CONFIG_NAME: 'mainnet' PRESET_BASE: 'mainnet' +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'mainnet' + # Transition # --------------------------------------------------------------- # Estimated on Sept 15, 2022 @@ -12,6 +18,8 @@ TERMINAL_TOTAL_DIFFICULTY: 58750000000000000000000 TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + + # Genesis # --------------------------------------------------------------- # `2**14` (= 16,384) @@ -32,22 +40,16 @@ GENESIS_DELAY: 604800 # Altair ALTAIR_FORK_VERSION: 0x01000000 -ALTAIR_FORK_EPOCH: 74240 -# Merge +ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC +# Bellatrix BELLATRIX_FORK_VERSION: 0x02000000 -BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC # Capella CAPELLA_FORK_VERSION: 0x03000000 -CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC +CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Deneb DENEB_FORK_VERSION: 0x04000000 -DENEB_FORK_EPOCH: 18446744073709551615 -# Sharding -SHARDING_FORK_VERSION: 0x03000000 -SHARDING_FORK_EPOCH: 18446744073709551615 - -# TBD, 2**32 is a placeholder. Merge transition approach is in active R&D. -TRANSITION_TOTAL_DIFFICULTY: 4294967296 +DENEB_FORK_EPOCH: 269568 # March 13, 2024, 01:55:35pm UTC # Time parameters @@ -74,16 +76,22 @@ INACTIVITY_SCORE_RECOVERY_RATE: 16 EJECTION_BALANCE: 16000000000 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 -# 2**3 (= 8) -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 - +# [New in Deneb:EIP7514] 2**3 (= 8) +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # Fork choice # --------------------------------------------------------------- # 40% PROPOSER_SCORE_BOOST: 40 +# 20% +REORG_HEAD_WEIGHT_THRESHOLD: 20 +# 160% +REORG_PARENT_WEIGHT_THRESHOLD: 160 +# `2` epochs +REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + # Deposit contract # --------------------------------------------------------------- @@ -92,17 +100,43 @@ DEPOSIT_CHAIN_ID: 1 DEPOSIT_NETWORK_ID: 1 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa -# Network + +# Networking # --------------------------------------------------------------- -SUBNETS_PER_NODE: 2 +# `10 * 2**20` (= 10485760, 10 MiB) GOSSIP_MAX_SIZE: 10485760 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `10 * 2**20` (=10485760, 10 MiB) MAX_CHUNK_SIZE: 10485760 +# 5s TTFB_TIMEOUT: 5 +# 10s RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 565b8d78902..a76a8320aa8 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -583,6 +583,8 @@ mod tests { } else { GenesisStateSource::Unknown }; + // With Deneb enabled by default we must set a trusted setup here. + let kzg_trusted_setup = get_trusted_setup_from_config(&config).unwrap(); let testnet = Eth2NetworkConfig { deposit_contract_deploy_block, @@ -593,7 +595,7 @@ mod tests { .map(Encode::as_ssz_bytes) .map(Into::into), config, - kzg_trusted_setup: None, + kzg_trusted_setup: Some(kzg_trusted_setup), }; testnet diff --git a/consensus/types/presets/mainnet/deneb.yaml b/consensus/types/presets/mainnet/deneb.yaml index 6d2fb4abde9..0f56b8bdfac 100644 --- a/consensus/types/presets/mainnet/deneb.yaml +++ b/consensus/types/presets/mainnet/deneb.yaml @@ -8,5 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 # `uint64(6)` MAX_BLOBS_PER_BLOCK: 6 -# `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 +# `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index a182c0f98d6..c32c67fa330 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -681,7 +681,7 @@ impl ChainSpec { * Deneb hard fork params */ deneb_fork_version: [0x04, 0x00, 0x00, 0x00], - deneb_fork_epoch: None, + deneb_fork_epoch: Some(Epoch::new(269568)), /* * Network specific diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index f97f17a6677..94996eb1a26 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -55,6 +55,12 @@ impl CommandLineTest { } fn run_with_zero_port(&mut self) -> CompletedTest { + // Required since Deneb was enabled on mainnet. + self.cmd.arg("--allow-insecure-genesis-sync"); + self.run_with_zero_port_and_no_genesis_sync() + } + + fn run_with_zero_port_and_no_genesis_sync(&mut self) -> CompletedTest { self.cmd.arg("-z"); self.run() } @@ -93,16 +99,16 @@ fn staking_flag() { } #[test] -fn allow_insecure_genesis_sync() { - CommandLineTest::new() - .run_with_zero_port() - .with_config(|config| { - assert_eq!(config.allow_insecure_genesis_sync, false); - }); +#[should_panic] +fn allow_insecure_genesis_sync_default() { + CommandLineTest::new().run_with_zero_port_and_no_genesis_sync(); +} +#[test] +fn allow_insecure_genesis_sync_enabled() { CommandLineTest::new() .flag("allow-insecure-genesis-sync", None) - .run_with_zero_port() + .run_with_zero_port_and_no_genesis_sync() .with_config(|config| { assert_eq!(config.allow_insecure_genesis_sync, true); }); @@ -851,6 +857,7 @@ fn network_port_flag_over_ipv4() { let port = 0; CommandLineTest::new() .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -867,6 +874,7 @@ fn network_port_flag_over_ipv4() { let port = 9000; CommandLineTest::new() .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -886,6 +894,7 @@ fn network_port_flag_over_ipv6() { CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -903,6 +912,7 @@ fn network_port_flag_over_ipv6() { CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -925,6 +935,7 @@ fn network_port_flag_over_ipv4_and_ipv6() { .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) .flag("port6", Some(port6.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -954,6 +965,7 @@ fn network_port_flag_over_ipv4_and_ipv6() { .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) .flag("port6", Some(port6.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -983,6 +995,7 @@ fn network_port_and_discovery_port_flags_over_ipv4() { CommandLineTest::new() .flag("port", Some(tcp4_port.to_string().as_str())) .flag("discovery-port", Some(disc4_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1003,6 +1016,7 @@ fn network_port_and_discovery_port_flags_over_ipv6() { .flag("listen-address", Some("::1")) .flag("port", Some(tcp6_port.to_string().as_str())) .flag("discovery-port", Some(disc6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1028,6 +1042,7 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { .flag("discovery-port", Some(disc4_port.to_string().as_str())) .flag("port6", Some(tcp6_port.to_string().as_str())) .flag("discovery-port6", Some(disc6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1067,6 +1082,7 @@ fn network_port_discovery_quic_port_flags_over_ipv4_and_ipv6() { .flag("port6", Some(tcp6_port.to_string().as_str())) .flag("discovery-port6", Some(disc6_port.to_string().as_str())) .flag("quic-port6", Some(quic6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1293,6 +1309,7 @@ fn enr_match_flag_over_ipv4() { .flag("listen-address", Some("127.0.0.2")) .flag("discovery-port", Some(udp4_port.to_string().as_str())) .flag("port", Some(tcp4_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1324,6 +1341,7 @@ fn enr_match_flag_over_ipv6() { .flag("listen-address", Some(ADDR)) .flag("discovery-port", Some(udp6_port.to_string().as_str())) .flag("port", Some(tcp6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1364,6 +1382,7 @@ fn enr_match_flag_over_ipv4_and_ipv6() { .flag("listen-address", Some(IPV6_ADDR)) .flag("discovery-port6", Some(udp6_port.to_string().as_str())) .flag("port6", Some(tcp6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1490,6 +1509,7 @@ fn http_port_flag() { .flag("http", None) .flag("http-port", Some(port1.to_string().as_str())) .flag("port", Some(port2.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| assert_eq!(config.http_api.listen_port, port1)); } @@ -1647,6 +1667,7 @@ fn metrics_port_flag() { .flag("metrics", None) .flag("metrics-port", Some(port1.to_string().as_str())) .flag("port", Some(port2.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| assert_eq!(config.http_metrics.listen_port, port1)); } From 49536ff103c2ec151950f41b61ed34ab59251076 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 15 Feb 2024 23:23:58 +1100 Subject: [PATCH 04/21] Add `distributed` flag to VC to enable support for DVT (#4867) * Initial flag building * Update validator_client/src/cli.rs Co-authored-by: Abhishek Kumar <43061995+xenowits@users.noreply.github.com> * Merge latest unstable * Per slot aggregates * One slot lookahead for sync committee aggregates * Update validator_client/src/duties_service.rs Co-authored-by: Abhishek Kumar <43061995+xenowits@users.noreply.github.com> * Rename selection_look_ahead * Merge branch 'unstable' into vc-distributed * Merge remote-tracking branch 'origin/unstable' into vc-distributed * Update CLI text --- book/src/help_vc.md | 3 + validator_client/Cargo.toml | 2 +- validator_client/src/cli.rs | 6 + validator_client/src/config.rs | 7 + validator_client/src/duties_service.rs | 22 +- validator_client/src/duties_service/sync.rs | 251 ++++++++++-------- validator_client/src/lib.rs | 5 +- .../src/sync_committee_service.rs | 4 +- 8 files changed, 173 insertions(+), 127 deletions(-) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index 02819804315..3d2519aac57 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -26,6 +26,9 @@ FLAGS: but is only safe if slashing protection is enabled on the remote signer and is implemented correctly. DO NOT ENABLE THIS FLAG UNLESS YOU ARE CERTAIN THAT SLASHING PROTECTION IS ENABLED ON THE REMOTE SIGNER. YOU WILL GET SLASHED IF YOU USE THIS FLAG WITHOUT ENABLING WEB3SIGNER'S SLASHING PROTECTION. + --distributed + Enables functionality required for running the validator in a distributed validator cluster. + --enable-doppelganger-protection If this flag is set, Lighthouse will delay startup for three epochs and monitor for messages on the network by any of the validators managed by this client. This will result in three (possibly four) epochs worth of diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 90a82b7e3b2..8e587c6155f 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [dev-dependencies] tokio = { workspace = true } +itertools = { workspace = true } [dependencies] tree_hash = { workspace = true } @@ -51,7 +52,6 @@ ring = { workspace = true } rand = { workspace = true, features = ["small_rng"] } lighthouse_metrics = { workspace = true } lazy_static = { workspace = true } -itertools = { workspace = true } monitoring_api = { workspace = true } sensitive_url = { workspace = true } task_executor = { workspace = true } diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index e4e5a2f1307..16a265212e5 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -145,6 +145,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { future.") .takes_value(false) ) + .arg( + Arg::with_name("distributed") + .long("distributed") + .help("Enables functionality required for running the validator in a distributed validator cluster.") + .takes_value(false) + ) /* REST API related arguments */ .arg( Arg::with_name("http") diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 7ac9e3e3bc7..ae59829a3e6 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -84,6 +84,8 @@ pub struct Config { pub builder_boost_factor: Option, /// If true, Lighthouse will prefer builder proposals, if available. pub prefer_builder_proposals: bool, + /// Whether we are running with distributed network support. + pub distributed: bool, pub web3_signer_keep_alive_timeout: Option, pub web3_signer_max_idle_connections: Option, } @@ -130,6 +132,7 @@ impl Default for Config { produce_block_v3: false, builder_boost_factor: None, prefer_builder_proposals: false, + distributed: false, web3_signer_keep_alive_timeout: Some(Duration::from_secs(90)), web3_signer_max_idle_connections: None, } @@ -233,6 +236,10 @@ impl Config { config.beacon_nodes_tls_certs = Some(tls_certs.split(',').map(PathBuf::from).collect()); } + if cli_args.is_present("distributed") { + config.distributed = true; + } + if cli_args.is_present("disable-run-on-all") { warn!( log, diff --git a/validator_client/src/duties_service.rs b/validator_client/src/duties_service.rs index 26747f81110..290803e257a 100644 --- a/validator_client/src/duties_service.rs +++ b/validator_client/src/duties_service.rs @@ -6,7 +6,7 @@ //! The `DutiesService` is also responsible for sending events to the `BlockService` which trigger //! block production. -mod sync; +pub mod sync; use crate::beacon_node_fallback::{ApiTopic, BeaconNodeFallback, OfflineOnFailure, RequireSynced}; use crate::http_metrics::metrics::{get_int_gauge, set_int_gauge, ATTESTATION_DUTY}; @@ -42,6 +42,9 @@ const HISTORICAL_DUTIES_EPOCHS: u64 = 2; /// At start-up selection proofs will be computed with less lookahead out of necessity. const SELECTION_PROOF_SLOT_LOOKAHEAD: u64 = 8; +/// The attestation selection proof lookahead for those running with the --distributed flag. +const SELECTION_PROOF_SLOT_LOOKAHEAD_DVT: u64 = 1; + /// Fraction of a slot at which selection proof signing should happen (2 means half way). const SELECTION_PROOF_SCHEDULE_DENOM: u32 = 2; @@ -211,16 +214,21 @@ pub struct DutiesService { /// proposals for any validators which are not registered locally. pub proposers: RwLock, /// Map from validator index to sync committee duties. - pub sync_duties: SyncDutiesMap, + pub sync_duties: SyncDutiesMap, /// Provides the canonical list of locally-managed validators. pub validator_store: Arc>, /// Tracks the current slot. pub slot_clock: T, /// Provides HTTP access to remote beacon nodes. pub beacon_nodes: Arc>, - pub enable_high_validator_count_metrics: bool, + /// The runtime for spawning tasks. pub context: RuntimeContext, + /// The current chain spec. pub spec: ChainSpec, + //// Whether we permit large validator counts in the metrics. + pub enable_high_validator_count_metrics: bool, + /// If this validator is running in distributed mode. + pub distributed: bool, } impl DutiesService { @@ -997,7 +1005,13 @@ async fn fill_in_selection_proofs( continue; }; - let lookahead_slot = current_slot + SELECTION_PROOF_SLOT_LOOKAHEAD; + let selection_lookahead = if duties_service.distributed { + SELECTION_PROOF_SLOT_LOOKAHEAD_DVT + } else { + SELECTION_PROOF_SLOT_LOOKAHEAD + }; + + let lookahead_slot = current_slot + selection_lookahead; let mut relevant_duties = duties_by_slot.split_off(&lookahead_slot); std::mem::swap(&mut relevant_duties, &mut duties_by_slot); diff --git a/validator_client/src/duties_service/sync.rs b/validator_client/src/duties_service/sync.rs index de42fa587ef..3618b47146f 100644 --- a/validator_client/src/duties_service/sync.rs +++ b/validator_client/src/duties_service/sync.rs @@ -7,18 +7,18 @@ use crate::{ }; use futures::future::join_all; -use itertools::Itertools; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use slog::{crit, debug, info, warn}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; +use std::marker::PhantomData; use std::sync::Arc; -use types::{ - ChainSpec, Epoch, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId, -}; +use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId}; -/// Number of epochs in advance to compute selection proofs. +/// Number of epochs in advance to compute selection proofs when not in `distributed` mode. pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; +/// Number of slots in advance to compute selection proofs when in `distributed` mode. +pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; /// Top-level data-structure containing sync duty information. /// @@ -32,9 +32,12 @@ pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; /// 2. One-at-a-time locking. For the innermost locks on the aggregator duties, all of the functions /// in this file take care to only lock one validator at a time. We never hold a lock while /// trying to obtain another one (hence no lock ordering issues). -pub struct SyncDutiesMap { +pub struct SyncDutiesMap { /// Map from sync committee period to duties for members of that sync committee. committees: RwLock>, + /// Whether we are in `distributed` mode and using reduced lookahead for aggregate pre-compute. + distributed: bool, + _phantom: PhantomData, } /// Duties for a single sync committee period. @@ -59,8 +62,8 @@ pub struct ValidatorDuties { /// Aggregator duties for a single validator. pub struct AggregatorDuties { - /// The epoch up to which aggregation proofs have already been computed (inclusive). - pre_compute_epoch: RwLock>, + /// The slot up to which aggregation proofs have already been computed (inclusive). + pre_compute_slot: RwLock>, /// Map from slot & subnet ID to proof that this validator is an aggregator. /// /// The slot is the slot at which the signed contribution and proof should be broadcast, @@ -82,15 +85,15 @@ pub struct SlotDuties { pub aggregators: HashMap>, } -impl Default for SyncDutiesMap { - fn default() -> Self { +impl SyncDutiesMap { + pub fn new(distributed: bool) -> Self { Self { committees: RwLock::new(HashMap::new()), + distributed, + _phantom: PhantomData, } } -} -impl SyncDutiesMap { /// Check if duties are already known for all of the given validators for `committee_period`. fn all_duties_known(&self, committee_period: u64, validator_indices: &[u64]) -> bool { self.committees @@ -104,22 +107,34 @@ impl SyncDutiesMap { }) } + /// Number of slots in advance to compute selection proofs + fn aggregation_pre_compute_slots(&self) -> u64 { + if self.distributed { + AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED + } else { + E::slots_per_epoch() * AGGREGATION_PRE_COMPUTE_EPOCHS + } + } + /// Prepare for pre-computation of selection proofs for `committee_period`. /// - /// Return the epoch up to which proofs should be pre-computed, as well as a vec of - /// `(previous_pre_compute_epoch, sync_duty)` pairs for all validators which need to have proofs + /// Return the slot up to which proofs should be pre-computed, as well as a vec of + /// `(previous_pre_compute_slot, sync_duty)` pairs for all validators which need to have proofs /// computed. See `fill_in_aggregation_proofs` for the actual calculation. fn prepare_for_aggregator_pre_compute( &self, committee_period: u64, - current_epoch: Epoch, + current_slot: Slot, spec: &ChainSpec, - ) -> (Epoch, Vec<(Epoch, SyncDuty)>) { - let default_start_epoch = - std::cmp::max(current_epoch, first_epoch_of_period(committee_period, spec)); - let pre_compute_epoch = std::cmp::min( - current_epoch + AGGREGATION_PRE_COMPUTE_EPOCHS, - last_epoch_of_period(committee_period, spec), + ) -> (Slot, Vec<(Slot, SyncDuty)>) { + let default_start_slot = std::cmp::max( + current_slot, + first_slot_of_period::(committee_period, spec), + ); + let pre_compute_lookahead_slots = self.aggregation_pre_compute_slots(); + let pre_compute_slot = std::cmp::min( + current_slot + pre_compute_lookahead_slots, + last_slot_of_period::(committee_period, spec), ); let pre_compute_duties = self.committees.read().get(&committee_period).map_or_else( @@ -130,18 +145,18 @@ impl SyncDutiesMap { .values() .filter_map(|maybe_duty| { let duty = maybe_duty.as_ref()?; - let old_pre_compute_epoch = duty + let old_pre_compute_slot = duty .aggregation_duties - .pre_compute_epoch + .pre_compute_slot .write() - .replace(pre_compute_epoch); + .replace(pre_compute_slot); - match old_pre_compute_epoch { + match old_pre_compute_slot { // No proofs pre-computed previously, compute all from the start of - // the period or the current epoch (whichever is later). - None => Some((default_start_epoch, duty.duty.clone())), + // the period or the current slot (whichever is later). + None => Some((default_start_slot, duty.duty.clone())), // Proofs computed up to `prev`, start from the subsequent epoch. - Some(prev) if prev < pre_compute_epoch => { + Some(prev) if prev < pre_compute_slot => { Some((prev + 1, duty.duty.clone())) } // Proofs already known, no need to compute. @@ -151,7 +166,7 @@ impl SyncDutiesMap { .collect() }, ); - (pre_compute_epoch, pre_compute_duties) + (pre_compute_slot, pre_compute_duties) } fn get_or_create_committee_duties<'a, 'b>( @@ -176,7 +191,7 @@ impl SyncDutiesMap { /// Get duties for all validators for the given `wall_clock_slot`. /// /// This is the entry-point for the sync committee service. - pub fn get_duties_for_slot( + pub fn get_duties_for_slot( &self, wall_clock_slot: Slot, spec: &ChainSpec, @@ -253,7 +268,7 @@ impl ValidatorDuties { Self { duty, aggregation_duties: AggregatorDuties { - pre_compute_epoch: RwLock::new(None), + pre_compute_slot: RwLock::new(None), proofs: RwLock::new(HashMap::new()), }, } @@ -265,12 +280,12 @@ fn epoch_offset(spec: &ChainSpec) -> u64 { spec.epochs_per_sync_committee_period.as_u64() / 2 } -fn first_epoch_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Epoch { - spec.epochs_per_sync_committee_period * sync_committee_period +fn first_slot_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Slot { + (spec.epochs_per_sync_committee_period * sync_committee_period).start_slot(E::slots_per_epoch()) } -fn last_epoch_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Epoch { - first_epoch_of_period(sync_committee_period + 1, spec) - 1 +fn last_slot_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Slot { + first_slot_of_period::(sync_committee_period + 1, spec) - 1 } pub async fn poll_sync_committee_duties( @@ -278,11 +293,11 @@ pub async fn poll_sync_committee_duties( ) -> Result<(), Error> { let sync_duties = &duties_service.sync_duties; let spec = &duties_service.spec; - let current_epoch = duties_service + let current_slot = duties_service .slot_clock .now() - .ok_or(Error::UnableToReadSlotClock)? - .epoch(E::slots_per_epoch()); + .ok_or(Error::UnableToReadSlotClock)?; + let current_epoch = current_slot.epoch(E::slots_per_epoch()); // If the Altair fork is yet to be activated, do not attempt to poll for duties. if spec @@ -330,8 +345,8 @@ pub async fn poll_sync_committee_duties( } // Pre-compute aggregator selection proofs for the current period. - let (current_pre_compute_epoch, new_pre_compute_duties) = sync_duties - .prepare_for_aggregator_pre_compute(current_sync_committee_period, current_epoch, spec); + let (current_pre_compute_slot, new_pre_compute_duties) = sync_duties + .prepare_for_aggregator_pre_compute(current_sync_committee_period, current_slot, spec); if !new_pre_compute_duties.is_empty() { let sub_duties_service = duties_service.clone(); @@ -341,8 +356,8 @@ pub async fn poll_sync_committee_duties( sub_duties_service, &new_pre_compute_duties, current_sync_committee_period, - current_epoch, - current_pre_compute_epoch, + current_slot, + current_pre_compute_slot, ) .await }, @@ -368,11 +383,14 @@ pub async fn poll_sync_committee_duties( } // Pre-compute aggregator selection proofs for the next period. - if (current_epoch + AGGREGATION_PRE_COMPUTE_EPOCHS).sync_committee_period(spec)? + let aggregate_pre_compute_lookahead_slots = sync_duties.aggregation_pre_compute_slots(); + if (current_slot + aggregate_pre_compute_lookahead_slots) + .epoch(E::slots_per_epoch()) + .sync_committee_period(spec)? == next_sync_committee_period { - let (pre_compute_epoch, new_pre_compute_duties) = sync_duties - .prepare_for_aggregator_pre_compute(next_sync_committee_period, current_epoch, spec); + let (pre_compute_slot, new_pre_compute_duties) = sync_duties + .prepare_for_aggregator_pre_compute(next_sync_committee_period, current_slot, spec); if !new_pre_compute_duties.is_empty() { let sub_duties_service = duties_service.clone(); @@ -382,8 +400,8 @@ pub async fn poll_sync_committee_duties( sub_duties_service, &new_pre_compute_duties, next_sync_committee_period, - current_epoch, - pre_compute_epoch, + current_slot, + pre_compute_slot, ) .await }, @@ -495,10 +513,10 @@ pub async fn poll_sync_committee_duties_for_period( duties_service: Arc>, - pre_compute_duties: &[(Epoch, SyncDuty)], + pre_compute_duties: &[(Slot, SyncDuty)], sync_committee_period: u64, - current_epoch: Epoch, - pre_compute_epoch: Epoch, + current_slot: Slot, + pre_compute_slot: Slot, ) { let log = duties_service.context.log(); @@ -506,16 +524,16 @@ pub async fn fill_in_aggregation_proofs( log, "Calculating sync selection proofs"; "period" => sync_committee_period, - "current_epoch" => current_epoch, - "pre_compute_epoch" => pre_compute_epoch + "current_slot" => current_slot, + "pre_compute_slot" => pre_compute_slot ); - // Generate selection proofs for each validator at each slot, one epoch at a time. - for epoch in (current_epoch.as_u64()..=pre_compute_epoch.as_u64()).map(Epoch::new) { + // Generate selection proofs for each validator at each slot, one slot at a time. + for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { let mut validator_proofs = vec![]; - for (validator_start_epoch, duty) in pre_compute_duties { - // Proofs are already known at this epoch for this validator. - if epoch < *validator_start_epoch { + for (validator_start_slot, duty) in pre_compute_duties { + // Proofs are already known at this slot for this validator. + if slot < *validator_start_slot { continue; } @@ -533,67 +551,64 @@ pub async fn fill_in_aggregation_proofs( // Create futures to produce proofs. let duties_service_ref = &duties_service; - let futures = epoch - .slot_iter(E::slots_per_epoch()) - .cartesian_product(&subnet_ids) - .map(|(duty_slot, subnet_id)| async move { - // Construct proof for prior slot. - let slot = duty_slot - 1; - - let proof = match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, slot, *subnet_id) - .await - { - Ok(proof) => proof, - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - debug!( - log, - "Missing pubkey for sync selection proof"; - "pubkey" => ?pubkey, - "pubkey" => ?duty.pubkey, - "slot" => slot, - ); - return None; - } - Err(e) => { - warn!( - log, - "Unable to sign selection proof"; - "error" => ?e, - "pubkey" => ?duty.pubkey, - "slot" => slot, - ); - return None; - } - }; - - match proof.is_aggregator::() { - Ok(true) => { - debug!( - log, - "Validator is sync aggregator"; - "validator_index" => duty.validator_index, - "slot" => slot, - "subnet_id" => %subnet_id, - ); - Some(((slot, *subnet_id), proof)) - } - Ok(false) => None, - Err(e) => { - warn!( - log, - "Error determining is_aggregator"; - "pubkey" => ?duty.pubkey, - "slot" => slot, - "error" => ?e, - ); - None - } + let futures = subnet_ids.iter().map(|subnet_id| async move { + // Construct proof for prior slot. + let proof_slot = slot - 1; + + let proof = match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + debug!( + log, + "Missing pubkey for sync selection proof"; + "pubkey" => ?pubkey, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; + } + Err(e) => { + warn!( + log, + "Unable to sign selection proof"; + "error" => ?e, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; } - }); + }; + + match proof.is_aggregator::() { + Ok(true) => { + debug!( + log, + "Validator is sync aggregator"; + "validator_index" => duty.validator_index, + "slot" => proof_slot, + "subnet_id" => %subnet_id, + ); + Some(((proof_slot, *subnet_id), proof)) + } + Ok(false) => None, + Err(e) => { + warn!( + log, + "Error determining is_aggregator"; + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + "error" => ?e, + ); + None + } + } + }); // Execute all the futures in parallel, collecting any successful results. let proofs = join_all(futures) @@ -635,7 +650,7 @@ pub async fn fill_in_aggregation_proofs( debug!( log, "Finished computing sync selection proofs"; - "epoch" => epoch, + "slot" => slot, "updated_validators" => num_validators_updated, ); } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 4828f43a0d8..52de95a3735 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -39,7 +39,7 @@ use account_utils::validator_definitions::ValidatorDefinitions; use attestation_service::{AttestationService, AttestationServiceBuilder}; use block_service::{BlockService, BlockServiceBuilder}; use clap::ArgMatches; -use duties_service::DutiesService; +use duties_service::{sync::SyncDutiesMap, DutiesService}; use environment::RuntimeContext; use eth2::{reqwest::ClientBuilder, types::Graffiti, BeaconNodeHttpClient, StatusCode, Timeouts}; use http_api::ApiSecret; @@ -451,13 +451,14 @@ impl ProductionValidatorClient { let duties_service = Arc::new(DutiesService { attesters: <_>::default(), proposers: <_>::default(), - sync_duties: <_>::default(), + sync_duties: SyncDutiesMap::new(config.distributed), slot_clock: slot_clock.clone(), beacon_nodes: beacon_nodes.clone(), validator_store: validator_store.clone(), spec: context.eth2_config.spec.clone(), context: duties_context, enable_high_validator_count_metrics: config.enable_high_validator_count_metrics, + distributed: config.distributed, }); // Update the metrics server. diff --git a/validator_client/src/sync_committee_service.rs b/validator_client/src/sync_committee_service.rs index 90b62cd3b44..f7abb3855a3 100644 --- a/validator_client/src/sync_committee_service.rs +++ b/validator_client/src/sync_committee_service.rs @@ -161,7 +161,7 @@ impl SyncCommitteeService { let Some(slot_duties) = self .duties_service .sync_duties - .get_duties_for_slot::(slot, &self.duties_service.spec) + .get_duties_for_slot(slot, &self.duties_service.spec) else { debug!(log, "No duties known for slot {}", slot); return Ok(()); @@ -548,7 +548,7 @@ impl SyncCommitteeService { match self .duties_service .sync_duties - .get_duties_for_slot::(duty_slot, spec) + .get_duties_for_slot(duty_slot, spec) { Some(duties) => subscriptions.extend(subscriptions_from_sync_duties( duties.duties, From f17fb291b7ff565e348920d77d605c66d2e113c1 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Feb 2024 23:24:47 +1100 Subject: [PATCH 05/21] Handle unknown head during attestation publishing (#5010) * Handle unknown head during attestation publishing * Merge remote-tracking branch 'origin/unstable' into queue-http-attestations * Simplify task spawner * Improve logging * Add a test * Improve error logging * Merge remote-tracking branch 'origin/unstable' into queue-http-attestations * Fix beta compiler warnings --- beacon_node/client/src/builder.rs | 4 + beacon_node/http_api/src/lib.rs | 151 ++------- .../http_api/src/publish_attestations.rs | 319 ++++++++++++++++++ beacon_node/http_api/src/task_spawner.rs | 27 +- beacon_node/http_api/src/test_utils.rs | 16 +- .../http_api/tests/interactive_tests.rs | 76 +++++ beacon_node/http_api/tests/tests.rs | 2 + 7 files changed, 455 insertions(+), 140 deletions(-) create mode 100644 beacon_node/http_api/src/publish_attestations.rs diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 444a277509f..de6e08cc37a 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -505,6 +505,7 @@ where network_senders: None, network_globals: None, beacon_processor_send: None, + beacon_processor_reprocess_send: None, eth1_service: Some(genesis_service.eth1_service.clone()), log: context.log().clone(), sse_logging_components: runtime_context.sse_logging_components.clone(), @@ -747,6 +748,9 @@ where network_globals: self.network_globals.clone(), eth1_service: self.eth1_service.clone(), beacon_processor_send: Some(beacon_processor_channels.beacon_processor_tx.clone()), + beacon_processor_reprocess_send: Some( + beacon_processor_channels.work_reprocessing_tx.clone(), + ), sse_logging_components: runtime_context.sse_logging_components.clone(), log: log.clone(), }); diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index ec3fd494d49..b39450d7354 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -16,6 +16,7 @@ mod database; mod metrics; mod produce_block; mod proposer_duties; +mod publish_attestations; mod publish_blocks; mod standard_block_rewards; mod state_id; @@ -35,7 +36,7 @@ use beacon_chain::{ validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes, WhenSlotSkipped, }; -use beacon_processor::BeaconProcessorSend; +use beacon_processor::{work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend}; pub use block_id::BlockId; use builder_states::get_next_withdrawals; use bytes::Bytes; @@ -129,6 +130,7 @@ pub struct Context { pub network_senders: Option>, pub network_globals: Option>>, pub beacon_processor_send: Option>, + pub beacon_processor_reprocess_send: Option>, pub eth1_service: Option, pub sse_logging_components: Option, pub log: Logger, @@ -534,6 +536,11 @@ pub fn serve( .filter(|_| config.enable_beacon_processor); let task_spawner_filter = warp::any().map(move || TaskSpawner::new(beacon_processor_send.clone())); + let beacon_processor_reprocess_send = ctx + .beacon_processor_reprocess_send + .clone() + .filter(|_| config.enable_beacon_processor); + let reprocess_send_filter = warp::any().map(move || beacon_processor_reprocess_send.clone()); let duplicate_block_status_code = ctx.config.duplicate_block_status_code; @@ -1756,140 +1763,26 @@ pub fn serve( .and(warp::path::end()) .and(warp_utils::json::json()) .and(network_tx_filter.clone()) + .and(reprocess_send_filter) .and(log_filter.clone()) .then( |task_spawner: TaskSpawner, chain: Arc>, attestations: Vec>, network_tx: UnboundedSender>, - log: Logger| { - task_spawner.blocking_json_task(Priority::P0, move || { - let seen_timestamp = timestamp_now(); - let mut failures = Vec::new(); - let mut num_already_known = 0; - - for (index, attestation) in attestations.as_slice().iter().enumerate() { - let attestation = match chain - .verify_unaggregated_attestation_for_gossip(attestation, None) - { - Ok(attestation) => attestation, - Err(AttnError::PriorAttestationKnown { .. }) => { - num_already_known += 1; - - // Skip to the next attestation since an attestation for this - // validator is already known in this epoch. - // - // There's little value for the network in validating a second - // attestation for another validator since it is either: - // - // 1. A duplicate. - // 2. Slashable. - // 3. Invalid. - // - // We are likely to get duplicates in the case where a VC is using - // fallback BNs. If the first BN actually publishes some/all of a - // batch of attestations but fails to respond in a timely fashion, - // the VC is likely to try publishing the attestations on another - // BN. That second BN may have already seen the attestations from - // the first BN and therefore indicate that the attestations are - // "already seen". An attestation that has already been seen has - // been published on the network so there's no actual error from - // the perspective of the user. - // - // It's better to prevent slashable attestations from ever - // appearing on the network than trying to slash validators, - // especially those validators connected to the local API. - // - // There might be *some* value in determining that this attestation - // is invalid, but since a valid attestation already it exists it - // appears that this validator is capable of producing valid - // attestations and there's no immediate cause for concern. - continue; - } - Err(e) => { - error!(log, - "Failure verifying attestation for gossip"; - "error" => ?e, - "request_index" => index, - "committee_index" => attestation.data.index, - "attestation_slot" => attestation.data.slot, - ); - failures.push(api_types::Failure::new( - index, - format!("Verification: {:?}", e), - )); - // skip to the next attestation so we do not publish this one to gossip - continue; - } - }; - - // Notify the validator monitor. - chain - .validator_monitor - .read() - .register_api_unaggregated_attestation( - seen_timestamp, - attestation.indexed_attestation(), - &chain.slot_clock, - ); - - publish_pubsub_message( - &network_tx, - PubsubMessage::Attestation(Box::new(( - attestation.subnet_id(), - attestation.attestation().clone(), - ))), - )?; - - let committee_index = attestation.attestation().data.index; - let slot = attestation.attestation().data.slot; - - if let Err(e) = chain.apply_attestation_to_fork_choice(&attestation) { - error!(log, - "Failure applying verified attestation to fork choice"; - "error" => ?e, - "request_index" => index, - "committee_index" => committee_index, - "slot" => slot, - ); - failures.push(api_types::Failure::new( - index, - format!("Fork choice: {:?}", e), - )); - }; - - if let Err(e) = chain.add_to_naive_aggregation_pool(&attestation) { - error!(log, - "Failure adding verified attestation to the naive aggregation pool"; - "error" => ?e, - "request_index" => index, - "committee_index" => committee_index, - "slot" => slot, - ); - failures.push(api_types::Failure::new( - index, - format!("Naive aggregation pool: {:?}", e), - )); - } - } - - if num_already_known > 0 { - debug!( - log, - "Some unagg attestations already known"; - "count" => num_already_known - ); - } - - if failures.is_empty() { - Ok(()) - } else { - Err(warp_utils::reject::indexed_bad_request( - "error processing attestations".to_string(), - failures, - )) - } - }) + reprocess_tx: Option>, + log: Logger| async move { + let result = crate::publish_attestations::publish_attestations( + task_spawner, + chain, + attestations, + network_tx, + reprocess_tx, + log, + ) + .await + .map(|()| warp::reply::json(&())); + task_spawner::convert_rejection(result).await }, ); diff --git a/beacon_node/http_api/src/publish_attestations.rs b/beacon_node/http_api/src/publish_attestations.rs new file mode 100644 index 00000000000..ed7f1ed17c9 --- /dev/null +++ b/beacon_node/http_api/src/publish_attestations.rs @@ -0,0 +1,319 @@ +//! Import attestations and publish them to the network. +//! +//! This module gracefully handles attestations to unknown blocks by requeuing them and then +//! efficiently waiting for them to finish reprocessing (using an async yield). +//! +//! The following comments relate to the handling of duplicate attestations (relocated here during +//! refactoring): +//! +//! Skip to the next attestation since an attestation for this +//! validator is already known in this epoch. +//! +//! There's little value for the network in validating a second +//! attestation for another validator since it is either: +//! +//! 1. A duplicate. +//! 2. Slashable. +//! 3. Invalid. +//! +//! We are likely to get duplicates in the case where a VC is using +//! fallback BNs. If the first BN actually publishes some/all of a +//! batch of attestations but fails to respond in a timely fashion, +//! the VC is likely to try publishing the attestations on another +//! BN. That second BN may have already seen the attestations from +//! the first BN and therefore indicate that the attestations are +//! "already seen". An attestation that has already been seen has +//! been published on the network so there's no actual error from +//! the perspective of the user. +//! +//! It's better to prevent slashable attestations from ever +//! appearing on the network than trying to slash validators, +//! especially those validators connected to the local API. +//! +//! There might be *some* value in determining that this attestation +//! is invalid, but since a valid attestation already it exists it +//! appears that this validator is capable of producing valid +//! attestations and there's no immediate cause for concern. +use crate::task_spawner::{Priority, TaskSpawner}; +use beacon_chain::{ + validator_monitor::timestamp_now, AttestationError, BeaconChain, BeaconChainError, + BeaconChainTypes, +}; +use beacon_processor::work_reprocessing_queue::{QueuedUnaggregate, ReprocessQueueMessage}; +use eth2::types::Failure; +use lighthouse_network::PubsubMessage; +use network::NetworkMessage; +use slog::{debug, error, warn, Logger}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{ + mpsc::{Sender, UnboundedSender}, + oneshot, +}; +use types::Attestation; + +// Error variants are only used in `Debug` and considered `dead_code` by the compiler. +#[derive(Debug)] +enum Error { + Validation(AttestationError), + Publication, + ForkChoice(#[allow(dead_code)] BeaconChainError), + AggregationPool(#[allow(dead_code)] AttestationError), + ReprocessDisabled, + ReprocessFull, + ReprocessTimeout, +} + +enum PublishAttestationResult { + Success, + AlreadyKnown, + Reprocessing(oneshot::Receiver>), + Failure(Error), +} + +fn verify_and_publish_attestation( + chain: &Arc>, + attestation: &Attestation, + seen_timestamp: Duration, + network_tx: &UnboundedSender>, + log: &Logger, +) -> Result<(), Error> { + let attestation = chain + .verify_unaggregated_attestation_for_gossip(attestation, None) + .map_err(Error::Validation)?; + + // Publish. + network_tx + .send(NetworkMessage::Publish { + messages: vec![PubsubMessage::Attestation(Box::new(( + attestation.subnet_id(), + attestation.attestation().clone(), + )))], + }) + .map_err(|_| Error::Publication)?; + + // Notify the validator monitor. + chain + .validator_monitor + .read() + .register_api_unaggregated_attestation( + seen_timestamp, + attestation.indexed_attestation(), + &chain.slot_clock, + ); + + let fc_result = chain.apply_attestation_to_fork_choice(&attestation); + let naive_aggregation_result = chain.add_to_naive_aggregation_pool(&attestation); + + if let Err(e) = &fc_result { + warn!( + log, + "Attestation invalid for fork choice"; + "err" => ?e, + ); + } + if let Err(e) = &naive_aggregation_result { + warn!( + log, + "Attestation invalid for aggregation"; + "err" => ?e + ); + } + + if let Err(e) = fc_result { + Err(Error::ForkChoice(e)) + } else if let Err(e) = naive_aggregation_result { + Err(Error::AggregationPool(e)) + } else { + Ok(()) + } +} + +pub async fn publish_attestations( + task_spawner: TaskSpawner, + chain: Arc>, + attestations: Vec>, + network_tx: UnboundedSender>, + reprocess_send: Option>, + log: Logger, +) -> Result<(), warp::Rejection> { + // Collect metadata about attestations which we'll use to report failures. We need to + // move the `attestations` vec into the blocking task, so this small overhead is unavoidable. + let attestation_metadata = attestations + .iter() + .map(|att| (att.data.slot, att.data.index)) + .collect::>(); + + // Gossip validate and publish attestations that can be immediately processed. + let seen_timestamp = timestamp_now(); + let inner_log = log.clone(); + let mut prelim_results = task_spawner + .blocking_task(Priority::P0, move || { + Ok(attestations + .into_iter() + .map(|attestation| { + match verify_and_publish_attestation( + &chain, + &attestation, + seen_timestamp, + &network_tx, + &inner_log, + ) { + Ok(()) => PublishAttestationResult::Success, + Err(Error::Validation(AttestationError::UnknownHeadBlock { + beacon_block_root, + })) => { + let Some(reprocess_tx) = &reprocess_send else { + return PublishAttestationResult::Failure(Error::ReprocessDisabled); + }; + // Re-process. + let (tx, rx) = oneshot::channel(); + let reprocess_chain = chain.clone(); + let reprocess_network_tx = network_tx.clone(); + let reprocess_log = inner_log.clone(); + let reprocess_fn = move || { + let result = verify_and_publish_attestation( + &reprocess_chain, + &attestation, + seen_timestamp, + &reprocess_network_tx, + &reprocess_log, + ); + // Ignore failure on the oneshot that reports the result. This + // shouldn't happen unless some catastrophe befalls the waiting + // thread which causes it to drop. + let _ = tx.send(result); + }; + let reprocess_msg = + ReprocessQueueMessage::UnknownBlockUnaggregate(QueuedUnaggregate { + beacon_block_root, + process_fn: Box::new(reprocess_fn), + }); + if reprocess_tx.try_send(reprocess_msg).is_err() { + PublishAttestationResult::Failure(Error::ReprocessFull) + } else { + PublishAttestationResult::Reprocessing(rx) + } + } + Err(Error::Validation(AttestationError::PriorAttestationKnown { + .. + })) => PublishAttestationResult::AlreadyKnown, + Err(e) => PublishAttestationResult::Failure(e), + } + }) + .map(Some) + .collect::>()) + }) + .await?; + + // Asynchronously wait for re-processing of attestations to unknown blocks. This avoids blocking + // any of the beacon processor workers while we wait for reprocessing. + let (reprocess_indices, reprocess_futures): (Vec<_>, Vec<_>) = prelim_results + .iter_mut() + .enumerate() + .filter_map(|(i, opt_result)| { + if let Some(PublishAttestationResult::Reprocessing(..)) = &opt_result { + let PublishAttestationResult::Reprocessing(rx) = opt_result.take()? else { + // Unreachable. + return None; + }; + Some((i, rx)) + } else { + None + } + }) + .unzip(); + let reprocess_results = futures::future::join_all(reprocess_futures).await; + + // Join everything back together and construct a response. + // This part should be quick so we just stay in the Tokio executor's async task. + for (i, reprocess_result) in reprocess_indices.into_iter().zip(reprocess_results) { + let Some(result_entry) = prelim_results.get_mut(i) else { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "prelim out of bounds", + "request_index" => i, + ); + continue; + }; + *result_entry = Some(match reprocess_result { + Ok(Ok(())) => PublishAttestationResult::Success, + // Attestation failed processing on re-process. + Ok(Err(Error::Validation(AttestationError::PriorAttestationKnown { .. }))) => { + PublishAttestationResult::AlreadyKnown + } + Ok(Err(e)) => PublishAttestationResult::Failure(e), + // Oneshot was dropped, indicating that the attestation either timed out in the + // reprocess queue or was dropped due to some error. + Err(_) => PublishAttestationResult::Failure(Error::ReprocessTimeout), + }); + } + + // Construct the response. + let mut failures = vec![]; + let mut num_already_known = 0; + + for (index, result) in prelim_results.iter().enumerate() { + match result { + Some(PublishAttestationResult::Success) => {} + Some(PublishAttestationResult::AlreadyKnown) => num_already_known += 1, + Some(PublishAttestationResult::Failure(e)) => { + if let Some((slot, committee_index)) = attestation_metadata.get(index) { + error!( + log, + "Failure verifying attestation for gossip"; + "error" => ?e, + "request_index" => index, + "committee_index" => committee_index, + "attestation_slot" => slot, + ); + failures.push(Failure::new(index, format!("{e:?}"))); + } else { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "out of bounds", + "request_index" => index + ); + failures.push(Failure::new(index, "metadata logic error".into())); + } + } + Some(PublishAttestationResult::Reprocessing(_)) => { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "reprocessing", + "request_index" => index + ); + failures.push(Failure::new(index, "reprocess logic error".into())); + } + None => { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "result is None", + "request_index" => index + ); + failures.push(Failure::new(index, "result logic error".into())); + } + } + } + + if num_already_known > 0 { + debug!( + log, + "Some unagg attestations already known"; + "count" => num_already_known + ); + } + + if failures.is_empty() { + Ok(()) + } else { + Err(warp_utils::reject::indexed_bad_request( + "error processing attestations".to_string(), + failures, + )) + } +} diff --git a/beacon_node/http_api/src/task_spawner.rs b/beacon_node/http_api/src/task_spawner.rs index 8768e057dac..cfee5e01ca0 100644 --- a/beacon_node/http_api/src/task_spawner.rs +++ b/beacon_node/http_api/src/task_spawner.rs @@ -60,11 +60,15 @@ impl TaskSpawner { } } - /// Executes a "blocking" (non-async) task which returns a `Response`. - pub async fn blocking_response_task(self, priority: Priority, func: F) -> Response + /// Executes a "blocking" (non-async) task which returns an arbitrary value. + pub async fn blocking_task( + self, + priority: Priority, + func: F, + ) -> Result where F: FnOnce() -> Result + Send + Sync + 'static, - T: Reply + Send + 'static, + T: Send + 'static, { if let Some(beacon_processor_send) = &self.beacon_processor_send { // Create a closure that will execute `func` and send the result to @@ -79,22 +83,31 @@ impl TaskSpawner { }; // Send the function to the beacon processor for execution at some arbitrary time. - let result = send_to_beacon_processor( + send_to_beacon_processor( beacon_processor_send, priority, BlockingOrAsync::Blocking(Box::new(process_fn)), rx, ) .await - .and_then(|x| x); - convert_rejection(result).await + .and_then(|x| x) } else { // There is no beacon processor so spawn a task directly on the // tokio executor. - convert_rejection(warp_utils::task::blocking_response_task(func).await).await + warp_utils::task::blocking_task(func).await } } + /// Executes a "blocking" (non-async) task which returns a `Response`. + pub async fn blocking_response_task(self, priority: Priority, func: F) -> Response + where + F: FnOnce() -> Result + Send + Sync + 'static, + T: Reply + Send + 'static, + { + let result = self.blocking_task(priority, func).await; + convert_rejection(result).await + } + /// Executes a "blocking" (non-async) task which returns a JSON-serializable /// object. pub async fn blocking_json_task(self, priority: Priority, func: F) -> Response diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index b87fdf6088f..c1313168bcd 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -35,6 +35,7 @@ pub const EXTERNAL_ADDR: &str = "/ip4/0.0.0.0/tcp/9000"; /// HTTP API tester that allows interaction with the underlying beacon chain harness. pub struct InteractiveTester { + pub ctx: Arc>>, pub harness: BeaconChainHarness>, pub client: BeaconNodeHttpClient, pub network_rx: NetworkReceivers, @@ -43,10 +44,11 @@ pub struct InteractiveTester { /// The result of calling `create_api_server`. /// /// Glue-type between `tests::ApiTester` and `InteractiveTester`. -pub struct ApiServer> { +pub struct ApiServer> { + pub ctx: Arc>, pub server: SFut, pub listening_socket: SocketAddr, - pub network_rx: NetworkReceivers, + pub network_rx: NetworkReceivers, pub local_enr: Enr, pub external_peer_id: PeerId, } @@ -90,6 +92,7 @@ impl InteractiveTester { let harness = harness_builder.build(); let ApiServer { + ctx, server, listening_socket, network_rx, @@ -114,6 +117,7 @@ impl InteractiveTester { ); Self { + ctx, harness, client, network_rx, @@ -125,7 +129,7 @@ pub async fn create_api_server( chain: Arc>, test_runtime: &TestRuntime, log: Logger, -) -> ApiServer> { +) -> ApiServer> { // Use port 0 to allocate a new unused port. let port = 0; @@ -187,6 +191,7 @@ pub async fn create_api_server( } = BeaconProcessorChannels::new(&beacon_processor_config); let beacon_processor_send = beacon_processor_tx; + let reprocess_send = work_reprocessing_tx.clone(); BeaconProcessor { network_globals: network_globals.clone(), executor: test_runtime.task_executor.clone(), @@ -216,14 +221,17 @@ pub async fn create_api_server( network_senders: Some(network_senders), network_globals: Some(network_globals), beacon_processor_send: Some(beacon_processor_send), + beacon_processor_reprocess_send: Some(reprocess_send), eth1_service: Some(eth1_service), sse_logging_components: None, log, }); - let (listening_socket, server) = crate::serve(ctx, test_runtime.task_executor.exit()).unwrap(); + let (listening_socket, server) = + crate::serve(ctx.clone(), test_runtime.task_executor.exit()).unwrap(); ApiServer { + ctx, server, listening_socket, network_rx: network_receivers, diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 6fb197b41ab..d63d04fcec5 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -4,6 +4,7 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy}, ChainConfig, }; +use beacon_processor::work_reprocessing_queue::ReprocessQueueMessage; use eth2::types::ProduceBlockV3Response; use eth2::types::{DepositContractData, StateId}; use execution_layer::{ForkchoiceState, PayloadAttributes}; @@ -840,3 +841,78 @@ pub async fn fork_choice_before_proposal() { // D's parent is B. assert_eq!(block_d.parent_root(), block_root_b.into()); } + +// Test that attestations to unknown blocks are requeued and processed when their block arrives. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn queue_attestations_from_http() { + let validator_count = 128; + let all_validators = (0..validator_count).collect::>(); + + let tester = InteractiveTester::::new(None, validator_count).await; + let harness = &tester.harness; + let client = tester.client.clone(); + + let num_initial = 5; + + // Slot of the block attested to. + let attestation_slot = Slot::new(num_initial) + 1; + + // Make some initial blocks. + harness.advance_slot(); + harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + harness.advance_slot(); + assert_eq!(harness.get_current_slot(), attestation_slot); + + // Make the attested-to block without applying it. + let pre_state = harness.get_current_state(); + let (block, post_state) = harness.make_block(pre_state, attestation_slot).await; + let block_root = block.0.canonical_root(); + + // Make attestations to the block and POST them to the beacon node on a background thread. + let attestations = harness + .make_unaggregated_attestations( + &all_validators, + &post_state, + block.0.state_root(), + block_root.into(), + attestation_slot, + ) + .into_iter() + .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) + .collect::>(); + + let attestation_future = tokio::spawn(async move { + client + .post_beacon_pool_attestations(&attestations) + .await + .expect("attestations should be processed successfully") + }); + + // In parallel, apply the block. We need to manually notify the reprocess queue, because the + // `beacon_chain` does not know about the queue and will not update it for us. + let parent_root = block.0.parent_root(); + harness + .process_block(attestation_slot, block_root, block) + .await + .unwrap(); + tester + .ctx + .beacon_processor_reprocess_send + .as_ref() + .unwrap() + .send(ReprocessQueueMessage::BlockImported { + block_root, + parent_root, + }) + .await + .unwrap(); + + attestation_future.await.unwrap(); +} diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 2d946f30921..a7ba2c1ab86 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -248,6 +248,7 @@ impl ApiTester { let log = null_logger().unwrap(); let ApiServer { + ctx: _, server, listening_socket, network_rx, @@ -341,6 +342,7 @@ impl ApiTester { let log = null_logger().unwrap(); let ApiServer { + ctx: _, server, listening_socket, network_rx, From 1711b80779d70122d9f98e3b3115ba67f93e26f2 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 15 Feb 2024 07:25:02 -0500 Subject: [PATCH 06/21] enable doppelganger tests for deneb (#5137) * enable doppelganger tests for deneb * comment out lcli install skip * Add sanity check * Merge remote-tracking branch 'origin/unstable' into deneb-doppelganger --- .github/workflows/test-suite.yml | 4 +- lcli/src/new_testnet.rs | 74 ++++++++++++++++++++++---------- scripts/local_testnet/geth.sh | 3 +- scripts/tests/genesis.json | 2 +- scripts/tests/vars.env | 9 ++-- 5 files changed, 62 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 77d631a7dab..28b8ec29e33 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -309,7 +309,9 @@ jobs: run: | make - name: Install lcli - if: env.SELF_HOSTED_RUNNERS == 'false' + # TODO: uncomment after the version of lcli in https://github.com/sigp/lighthouse/pull/5137 + # is installed on the runners + # if: env.SELF_HOSTED_RUNNERS == 'false' run: make install-lcli - name: Run the doppelganger protection failure test script run: | diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index 3a0c7a9f60b..47db1036d98 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -9,7 +9,9 @@ use ethereum_hashing::hash; use ssz::Decode; use ssz::Encode; use state_processing::process_activations; -use state_processing::upgrade::{upgrade_to_altair, upgrade_to_bellatrix}; +use state_processing::upgrade::{ + upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, +}; use std::fs::File; use std::io::Read; use std::path::PathBuf; @@ -19,8 +21,8 @@ use types::ExecutionBlockHash; use types::{ test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Epoch, Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, - ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRefMut, - ForkName, Hash256, Keypair, PublicKey, Validator, + ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ForkName, Hash256, Keypair, + PublicKey, Validator, }; pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Result<(), String> { @@ -302,26 +304,47 @@ fn initialize_state_with_validators( state.fork_mut().previous_version = spec.bellatrix_fork_version; // Override latest execution payload header. - // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/merge/beacon-chain.md#testing - - // Currently, we only support starting from a bellatrix state - match state - .latest_execution_payload_header_mut() - .map_err(|e| format!("Failed to get execution payload header: {:?}", e))? - { - ExecutionPayloadHeaderRefMut::Merge(header_mut) => { - if let ExecutionPayloadHeader::Merge(eph) = execution_payload_header { - *header_mut = eph; - } else { - return Err("Execution payload header must be a bellatrix header".to_string()); - } - } - ExecutionPayloadHeaderRefMut::Capella(_) => { - return Err("Cannot start genesis from a capella state".to_string()) - } - ExecutionPayloadHeaderRefMut::Deneb(_) => { - return Err("Cannot start genesis from a deneb state".to_string()) - } + // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing + if let ExecutionPayloadHeader::Merge(ref header) = execution_payload_header { + *state + .latest_execution_payload_header_merge_mut() + .or(Err("mismatched fork".to_string()))? = header.clone(); + } + } + + if spec + .capella_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_capella(&mut state, spec).unwrap(); + + // Remove intermediate fork from `state.fork`. + state.fork_mut().previous_version = spec.capella_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing + if let ExecutionPayloadHeader::Capella(ref header) = execution_payload_header { + *state + .latest_execution_payload_header_capella_mut() + .or(Err("mismatched fork".to_string()))? = header.clone(); + } + } + + if spec + .deneb_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_deneb(&mut state, spec).unwrap(); + + // Remove intermediate fork from `state.fork`. + state.fork_mut().previous_version = spec.deneb_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing + if let ExecutionPayloadHeader::Deneb(ref header) = execution_payload_header { + *state + .latest_execution_payload_header_deneb_mut() + .or(Err("mismatched fork".to_string()))? = header.clone(); } } @@ -331,5 +354,10 @@ fn initialize_state_with_validators( // Set genesis validators root for domain separation and chain versioning *state.genesis_validators_root_mut() = state.update_validators_tree_hash_cache().unwrap(); + // Sanity check for state fork matching config fork. + state + .fork_name(spec) + .map_err(|e| format!("state fork mismatch: {e:?}"))?; + Ok(state) } diff --git a/scripts/local_testnet/geth.sh b/scripts/local_testnet/geth.sh index 5dc4575cf0a..ab1a0ec6ee0 100755 --- a/scripts/local_testnet/geth.sh +++ b/scripts/local_testnet/geth.sh @@ -50,4 +50,5 @@ exec $GETH_BINARY \ --bootnodes $EL_BOOTNODE_ENODE \ --port $network_port \ --http.port $http_port \ - --authrpc.port $auth_port + --authrpc.port $auth_port \ + 2>&1 | tee $data_dir/geth.log diff --git a/scripts/tests/genesis.json b/scripts/tests/genesis.json index 83f45f1a012..bfbc08c81e5 100644 --- a/scripts/tests/genesis.json +++ b/scripts/tests/genesis.json @@ -13,7 +13,7 @@ "londonBlock": 0, "mergeForkBlock": 0, "shanghaiTime": 0, - "shardingForkTime": 0, + "cancunTime": 0, "terminalTotalDifficulty": 0, "terminalTotalDifficultyPassed": true }, diff --git a/scripts/tests/vars.env b/scripts/tests/vars.env index 98ae08f0747..ffe7ac4aecd 100644 --- a/scripts/tests/vars.env +++ b/scripts/tests/vars.env @@ -16,7 +16,7 @@ DEPOSIT_CONTRACT_ADDRESS=4242424242424242424242424242424242424242 GENESIS_FORK_VERSION=0x42424242 # Block hash generated from genesis.json in directory -ETH1_BLOCK_HASH=add7865f8346031c72287e2edc4a4952fd34fc0a8642403e8c1bce67f215c92b +ETH1_BLOCK_HASH=7a5c656343c3a66dcf75415958b500e8873f9dab0cd588e6cf0785b52a06dd34 VALIDATOR_COUNT=80 GENESIS_VALIDATOR_COUNT=80 @@ -41,8 +41,8 @@ CHAIN_ID=4242 # Hard fork configuration ALTAIR_FORK_EPOCH=0 BELLATRIX_FORK_EPOCH=0 -CAPELLA_FORK_EPOCH=1 -DENEB_FORK_EPOCH=18446744073709551615 +CAPELLA_FORK_EPOCH=0 +DENEB_FORK_EPOCH=0 TTD=0 @@ -62,4 +62,5 @@ PROPOSER_SCORE_BOOST=70 BN_ARGS="" # Enable doppelganger detection -VC_ARGS=" --enable-doppelganger-protection " \ No newline at end of file +VC_ARGS=" --enable-doppelganger-protection " + From a264afd19ff1d8d4a9ce1e673386722ccb0e31ec Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Sun, 18 Feb 2024 23:40:45 +1100 Subject: [PATCH 07/21] Verify Versioned Hashes During Optimistic Sync (#4832) * Convert NewPayloadRequest to use Reference * Refactor for Clarity * Verify Versioned Hashes * Added Tests for Version Hash Verification * Added Moar Tests * Fix Problems Caused By Merge * Update to use Alloy Instead of Reth Crates (#14) * Update beacon_node/execution_layer/src/engine_api/new_payload_request.rs Co-authored-by: realbigsean * Faster Versioned Hash Extraction * Update to rust 1.75 & Pin alloy-consensus --- Cargo.lock | 423 +++++++++++++++++- Dockerfile | 2 +- .../beacon_chain/src/execution_payload.rs | 19 +- beacon_node/execution_layer/Cargo.toml | 2 + beacon_node/execution_layer/src/block_hash.rs | 125 ++---- beacon_node/execution_layer/src/engine_api.rs | 115 +---- .../execution_layer/src/engine_api/http.rs | 6 +- .../src/engine_api/new_payload_request.rs | 332 ++++++++++++++ beacon_node/execution_layer/src/lib.rs | 4 +- .../test_utils/execution_block_generator.rs | 2 +- .../src/test_utils/mock_execution_layer.rs | 2 +- .../execution_layer/src/test_utils/mod.rs | 4 +- .../execution_layer/src/versioned_hashes.rs | 135 ++++++ lcli/Dockerfile | 2 +- lighthouse/Cargo.toml | 2 +- .../src/test_rig.rs | 12 +- 16 files changed, 961 insertions(+), 226 deletions(-) create mode 100644 beacon_node/execution_layer/src/engine_api/new_payload_request.rs create mode 100644 beacon_node/execution_layer/src/versioned_hashes.rs diff --git a/Cargo.lock b/Cargo.lock index 83673d239c0..1c32fed0874 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,6 +179,95 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "alloy-consensus" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-eips", + "alloy-network", + "alloy-primitives", + "alloy-rlp", +] + +[[package]] +name = "alloy-eips" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "serde", + "thiserror", +] + +[[package]] +name = "alloy-json-rpc" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-primitives", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "alloy-network" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-eips", + "alloy-json-rpc", + "alloy-primitives", + "alloy-rlp", + "serde", +] + +[[package]] +name = "alloy-primitives" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4b6fb2b432ff223d513db7f908937f63c252bee0af9b82bfd25b0a5dd1eb0d8" +dependencies = [ + "alloy-rlp", + "bytes", + "cfg-if", + "const-hex", + "derive_more", + "hex-literal", + "itoa", + "k256 0.13.3", + "keccak-asm", + "proptest", + "rand", + "ruint", + "serde", + "tiny-keccak", +] + +[[package]] +name = "alloy-rlp" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d58d9f5da7b40e9bfff0b7e7816700be4019db97d4b6359fe7f94a9e22e42ac" +dependencies = [ + "alloy-rlp-derive", + "arrayvec", + "bytes", +] + +[[package]] +name = "alloy-rlp-derive" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a047897373be4bbb0224c1afdabca92648dc57a9c9ef6e7b0be3aff7a859c83" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "amcl" version = "0.3.0" @@ -229,6 +318,130 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "ark-ff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b3235cc41ee7a12aaaf2c575a2ad7b46713a8a50bda2fc3b003a04845c05dd6" +dependencies = [ + "ark-ff-asm 0.3.0", + "ark-ff-macros 0.3.0", + "ark-serialize 0.3.0", + "ark-std 0.3.0", + "derivative", + "num-bigint", + "num-traits", + "paste", + "rustc_version 0.3.3", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba" +dependencies = [ + "ark-ff-asm 0.4.2", + "ark-ff-macros 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "digest 0.10.7", + "itertools", + "num-bigint", + "num-traits", + "paste", + "rustc_version 0.4.0", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db02d390bf6643fb404d3d22d31aee1c4bc4459600aef9113833d17e786c6e44" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-asm" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" +dependencies = [ + "num-bigint", + "num-traits", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-serialize" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6c2b318ee6e10f8c2853e73a83adc0ccb88995aa978d8a3408d492ab2ee671" +dependencies = [ + "ark-std 0.3.0", + "digest 0.9.0", +] + +[[package]] +name = "ark-serialize" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" +dependencies = [ + "ark-std 0.4.0", + "digest 0.10.7", + "num-bigint", +] + +[[package]] +name = "ark-std" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "ark-std" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "arrayref" version = "0.3.7" @@ -485,7 +698,7 @@ checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ "futures", "pharos", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -838,6 +1051,21 @@ dependencies = [ "which", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -1119,7 +1347,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver", + "semver 1.0.21", "serde", "serde_json", "thiserror", @@ -1331,6 +1559,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-hex" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5104de16b218eddf8e34ffe2f86f74bfa4e61e95a1b89732fccf6325efd0557" +dependencies = [ + "cfg-if", + "cpufeatures", + "hex", + "proptest", + "serde", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1593,7 +1834,7 @@ dependencies = [ "digest 0.10.7", "fiat-crypto", "platforms 3.3.0", - "rustc_version", + "rustc_version 0.4.0", "subtle", "zeroize", ] @@ -1820,7 +2061,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn 1.0.109", ] @@ -2755,6 +2996,8 @@ dependencies = [ name = "execution_layer" version = "0.1.0" dependencies = [ + "alloy-consensus", + "alloy-rlp", "arc-swap", "async-trait", "builder_client", @@ -2848,6 +3091,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fastrlp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + [[package]] name = "ff" version = "0.12.1" @@ -2887,7 +3141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" dependencies = [ "memoffset", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -3454,6 +3708,15 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" [[package]] name = "hex_fmt" @@ -4188,6 +4451,16 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "keccak-asm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb8515fff80ed850aea4a1595f2e519c003e2a00a82fe168ebf5269196caf444" +dependencies = [ + "digest 0.10.7", + "sha3-asm", +] + [[package]] name = "keccak-hash" version = "0.10.0" @@ -5618,6 +5891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -5986,6 +6260,17 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + [[package]] name = "pharos" version = "0.5.3" @@ -5993,7 +6278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" dependencies = [ "futures", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -6373,6 +6658,26 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "proptest" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.4.2", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.8.2", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "proto_array" version = "0.2.0" @@ -6865,6 +7170,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "ruint" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608a5726529f2f0ef81b8fde9873c4bb829d6b5b5ca6be4d97345ddf0749c825" +dependencies = [ + "alloy-rlp", + "ark-ff 0.3.0", + "ark-ff 0.4.2", + "bytes", + "fastrlp", + "num-bigint", + "num-traits", + "parity-scale-codec 3.6.9", + "primitive-types 0.12.2", + "proptest", + "rand", + "rlp", + "ruint-macro", + "serde", + "valuable", + "zeroize", +] + +[[package]] +name = "ruint-macro" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e666a5496a0b2186dbcd0ff6106e29e093c15591bde62c20d3842007c6978a09" + [[package]] name = "rusqlite" version = "0.28.0" @@ -6897,13 +7232,22 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver", + "semver 1.0.21", ] [[package]] @@ -6993,6 +7337,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "rw-stream-sink" version = "0.4.0" @@ -7159,6 +7515,15 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.21" @@ -7168,6 +7533,15 @@ dependencies = [ "serde", ] +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + [[package]] name = "send_wrapper" version = "0.6.0" @@ -7367,6 +7741,16 @@ dependencies = [ "keccak", ] +[[package]] +name = "sha3-asm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac61da6b35ad76b195eb4771210f947734321a8d81d7738e1580d953bc7a15e" +dependencies = [ + "cc", + "cfg-if", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -7659,7 +8043,7 @@ dependencies = [ "curve25519-dalek", "rand_core", "ring 0.17.7", - "rustc_version", + "rustc_version 0.4.0", "sha2 0.10.8", "subtle", ] @@ -8621,6 +9005,12 @@ dependencies = [ "tree_hash_derive", ] +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "uint" version = "0.9.5" @@ -8634,6 +9024,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unescape" version = "0.1.0" @@ -8911,6 +9307,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.1.1" @@ -9496,7 +9901,7 @@ dependencies = [ "js-sys", "log", "pharos", - "rustc_version", + "rustc_version 0.4.0", "send_wrapper", "thiserror", "wasm-bindgen", diff --git a/Dockerfile b/Dockerfile index a8dadf2ad57..901c1b83d63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.73.0-bullseye AS builder +FROM rust:1.75.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index e25976c2a57..fd790c88429 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -81,14 +81,10 @@ impl PayloadNotifier { match notify_execution_layer { NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => { - // Verify the block hash here in Lighthouse and immediately mark the block as - // optimistically imported. This saves a lot of roundtrips to the EL. - let execution_layer = chain - .execution_layer - .as_ref() - .ok_or(ExecutionPayloadError::NoExecutionConnection)?; - - if let Err(e) = execution_layer.verify_payload_block_hash(block_message) { + // Create a NewPayloadRequest (no clones required) and check optimistic sync verifications + let new_payload_request: NewPayloadRequest = + block_message.try_into()?; + if let Err(e) = new_payload_request.perform_optimistic_sync_verifications() { warn!( chain.log, "Falling back to slow block hash verification"; @@ -143,11 +139,8 @@ async fn notify_new_payload<'a, T: BeaconChainTypes>( .as_ref() .ok_or(ExecutionPayloadError::NoExecutionConnection)?; - let new_payload_request: NewPayloadRequest = block.try_into()?; - let execution_block_hash = new_payload_request.block_hash(); - let new_payload_response = execution_layer - .notify_new_payload(new_payload_request) - .await; + let execution_block_hash = block.execution_payload()?.block_hash(); + let new_payload_response = execution_layer.notify_new_payload(block.try_into()?).await; match new_payload_response { Ok(status) => match status { diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index 7f652689806..7fee3721d8f 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -51,3 +51,5 @@ hash-db = "0.15.2" pretty_reqwest_error = { workspace = true } arc-swap = "1.6.0" eth2_network_config = { workspace = true } +alloy-rlp = "0.3" +alloy-consensus = { git = "https://github.com/alloy-rs/alloy.git", rev = "974d488bab5e21e9f17452a39a4bfa56677367b2" } diff --git a/beacon_node/execution_layer/src/block_hash.rs b/beacon_node/execution_layer/src/block_hash.rs index 5ba61beafca..074ef8b0c14 100644 --- a/beacon_node/execution_layer/src/block_hash.rs +++ b/beacon_node/execution_layer/src/block_hash.rs @@ -1,92 +1,61 @@ use crate::{ json_structures::JsonWithdrawal, keccak::{keccak256, KeccakHasher}, - metrics, Error, ExecutionLayer, }; use ethers_core::utils::rlp::RlpStream; use keccak_hash::KECCAK_EMPTY_LIST_RLP; use triehash::ordered_trie_root; use types::{ - map_execution_block_header_fields_base, Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, + map_execution_block_header_fields_base, Address, EthSpec, ExecutionBlockHash, ExecutionBlockHeader, ExecutionPayloadRef, Hash256, Hash64, Uint256, }; -impl ExecutionLayer { - /// Calculate the block hash of an execution block. - /// - /// Return `(block_hash, transactions_root)`, where `transactions_root` is the root of the RLP - /// transactions. - pub fn calculate_execution_block_hash( - payload: ExecutionPayloadRef, - parent_beacon_block_root: Hash256, - ) -> (ExecutionBlockHash, Hash256) { - // Calculate the transactions root. - // We're currently using a deprecated Parity library for this. We should move to a - // better alternative when one appears, possibly following Reth. - let rlp_transactions_root = ordered_trie_root::( - payload.transactions().iter().map(|txn_bytes| &**txn_bytes), - ); - - // Calculate withdrawals root (post-Capella). - let rlp_withdrawals_root = if let Ok(withdrawals) = payload.withdrawals() { - Some(ordered_trie_root::( - withdrawals.iter().map(|withdrawal| { - rlp_encode_withdrawal(&JsonWithdrawal::from(withdrawal.clone())) - }), - )) - } else { - None - }; - - let rlp_blob_gas_used = payload.blob_gas_used().ok(); - let rlp_excess_blob_gas = payload.excess_blob_gas().ok(); - - // Calculate parent beacon block root (post-Deneb). - let rlp_parent_beacon_block_root = rlp_excess_blob_gas - .as_ref() - .map(|_| parent_beacon_block_root); - - // Construct the block header. - let exec_block_header = ExecutionBlockHeader::from_payload( - payload, - KECCAK_EMPTY_LIST_RLP.as_fixed_bytes().into(), - rlp_transactions_root, - rlp_withdrawals_root, - rlp_blob_gas_used, - rlp_excess_blob_gas, - rlp_parent_beacon_block_root, - ); - - // Hash the RLP encoding of the block header. - let rlp_block_header = rlp_encode_block_header(&exec_block_header); - ( - ExecutionBlockHash::from_root(keccak256(&rlp_block_header)), - rlp_transactions_root, - ) - } - - /// Verify `payload.block_hash` locally within Lighthouse. - /// - /// No remote calls to the execution client will be made, so this is quite a cheap check. - pub fn verify_payload_block_hash(&self, block: BeaconBlockRef) -> Result<(), Error> { - let payload = block.execution_payload()?.execution_payload_ref(); - let parent_beacon_block_root = block.parent_root(); - - let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_VERIFY_BLOCK_HASH); - - let (header_hash, rlp_transactions_root) = - Self::calculate_execution_block_hash(payload, parent_beacon_block_root); - - if header_hash != payload.block_hash() { - return Err(Error::BlockHashMismatch { - computed: header_hash, - payload: payload.block_hash(), - transactions_root: rlp_transactions_root, - }); - } - - Ok(()) - } +/// Calculate the block hash of an execution block. +/// +/// Return `(block_hash, transactions_root)`, where `transactions_root` is the root of the RLP +/// transactions. +pub fn calculate_execution_block_hash( + payload: ExecutionPayloadRef, + parent_beacon_block_root: Option, +) -> (ExecutionBlockHash, Hash256) { + // Calculate the transactions root. + // We're currently using a deprecated Parity library for this. We should move to a + // better alternative when one appears, possibly following Reth. + let rlp_transactions_root = ordered_trie_root::( + payload.transactions().iter().map(|txn_bytes| &**txn_bytes), + ); + + // Calculate withdrawals root (post-Capella). + let rlp_withdrawals_root = if let Ok(withdrawals) = payload.withdrawals() { + Some(ordered_trie_root::( + withdrawals + .iter() + .map(|withdrawal| rlp_encode_withdrawal(&JsonWithdrawal::from(withdrawal.clone()))), + )) + } else { + None + }; + + let rlp_blob_gas_used = payload.blob_gas_used().ok(); + let rlp_excess_blob_gas = payload.excess_blob_gas().ok(); + + // Construct the block header. + let exec_block_header = ExecutionBlockHeader::from_payload( + payload, + KECCAK_EMPTY_LIST_RLP.as_fixed_bytes().into(), + rlp_transactions_root, + rlp_withdrawals_root, + rlp_blob_gas_used, + rlp_excess_blob_gas, + parent_beacon_block_root, + ); + + // Hash the RLP encoding of the block header. + let rlp_block_header = rlp_encode_block_header(&exec_block_header); + ( + ExecutionBlockHash::from_root(keccak256(&rlp_block_header)), + rlp_transactions_root, + ) } /// RLP encode a withdrawal. diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 19b9a58eb66..e20009e2858 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -17,7 +17,6 @@ pub use json_structures::{JsonWithdrawal, TransitionConfigurationV1}; use pretty_reqwest_error::PrettyReqwestError; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; -use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::convert::TryFrom; use strum::IntoStaticStr; use superstruct::superstruct; @@ -26,14 +25,16 @@ pub use types::{ ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; -use types::{ - BeaconStateError, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, - KzgProofs, VersionedHash, -}; +use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, KzgProofs}; pub mod auth; pub mod http; pub mod json_structures; +mod new_payload_request; + +pub use new_payload_request::{ + NewPayloadRequest, NewPayloadRequestCapella, NewPayloadRequestDeneb, NewPayloadRequestMerge, +}; pub const LATEST_TAG: &str = "latest"; @@ -571,110 +572,6 @@ impl ExecutionPayloadBodyV1 { } } -#[superstruct( - variants(Merge, Capella, Deneb), - variant_attributes(derive(Clone, Debug, PartialEq),), - map_into(ExecutionPayload), - map_ref_into(ExecutionPayloadRef), - cast_error( - ty = "BeaconStateError", - expr = "BeaconStateError::IncorrectStateVariant" - ), - partial_getter_error( - ty = "BeaconStateError", - expr = "BeaconStateError::IncorrectStateVariant" - ) -)] -#[derive(Clone, Debug, PartialEq)] -pub struct NewPayloadRequest { - #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] - pub execution_payload: ExecutionPayloadMerge, - #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] - pub execution_payload: ExecutionPayloadCapella, - #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] - pub execution_payload: ExecutionPayloadDeneb, - #[superstruct(only(Deneb))] - pub versioned_hashes: Vec, - #[superstruct(only(Deneb))] - pub parent_beacon_block_root: Hash256, -} - -impl NewPayloadRequest { - pub fn parent_hash(&self) -> ExecutionBlockHash { - match self { - Self::Merge(payload) => payload.execution_payload.parent_hash, - Self::Capella(payload) => payload.execution_payload.parent_hash, - Self::Deneb(payload) => payload.execution_payload.parent_hash, - } - } - - pub fn block_hash(&self) -> ExecutionBlockHash { - match self { - Self::Merge(payload) => payload.execution_payload.block_hash, - Self::Capella(payload) => payload.execution_payload.block_hash, - Self::Deneb(payload) => payload.execution_payload.block_hash, - } - } - - pub fn block_number(&self) -> u64 { - match self { - Self::Merge(payload) => payload.execution_payload.block_number, - Self::Capella(payload) => payload.execution_payload.block_number, - Self::Deneb(payload) => payload.execution_payload.block_number, - } - } - - pub fn into_execution_payload(self) -> ExecutionPayload { - map_new_payload_request_into_execution_payload!(self, |request, cons| { - cons(request.execution_payload) - }) - } -} - -impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest { - type Error = BeaconStateError; - - fn try_from(block: BeaconBlockRef<'a, E>) -> Result { - match block { - BeaconBlockRef::Base(_) | BeaconBlockRef::Altair(_) => { - Err(Self::Error::IncorrectStateVariant) - } - BeaconBlockRef::Merge(block_ref) => Ok(Self::Merge(NewPayloadRequestMerge { - execution_payload: block_ref.body.execution_payload.execution_payload.clone(), - })), - BeaconBlockRef::Capella(block_ref) => Ok(Self::Capella(NewPayloadRequestCapella { - execution_payload: block_ref.body.execution_payload.execution_payload.clone(), - })), - BeaconBlockRef::Deneb(block_ref) => Ok(Self::Deneb(NewPayloadRequestDeneb { - execution_payload: block_ref.body.execution_payload.execution_payload.clone(), - versioned_hashes: block_ref - .body - .blob_kzg_commitments - .iter() - .map(kzg_commitment_to_versioned_hash) - .collect(), - parent_beacon_block_root: block_ref.parent_root, - })), - } - } -} - -impl TryFrom> for NewPayloadRequest { - type Error = BeaconStateError; - - fn try_from(payload: ExecutionPayload) -> Result { - match payload { - ExecutionPayload::Merge(payload) => Ok(Self::Merge(NewPayloadRequestMerge { - execution_payload: payload, - })), - ExecutionPayload::Capella(payload) => Ok(Self::Capella(NewPayloadRequestCapella { - execution_payload: payload, - })), - ExecutionPayload::Deneb(_) => Err(Self::Error::IncorrectStateVariant), - } - } -} - #[derive(Clone, Copy, Debug)] pub struct EngineCapabilities { pub new_payload_v1: bool, diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index ac7dfa57e92..df0f79c61e2 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -803,10 +803,10 @@ impl HttpJsonRpc { pub async fn new_payload_v3( &self, - new_payload_request_deneb: NewPayloadRequestDeneb, + new_payload_request_deneb: NewPayloadRequestDeneb<'_, T>, ) -> Result { let params = json!([ - JsonExecutionPayload::V3(new_payload_request_deneb.execution_payload.into()), + JsonExecutionPayload::V3(new_payload_request_deneb.execution_payload.clone().into()), new_payload_request_deneb.versioned_hashes, new_payload_request_deneb.parent_beacon_block_root, ]); @@ -1079,7 +1079,7 @@ impl HttpJsonRpc { // new_payload that the execution engine supports pub async fn new_payload( &self, - new_payload_request: NewPayloadRequest, + new_payload_request: NewPayloadRequest<'_, T>, ) -> Result { let engine_capabilities = self.get_engine_capabilities(None).await?; match new_payload_request { diff --git a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs new file mode 100644 index 00000000000..b1385399e89 --- /dev/null +++ b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs @@ -0,0 +1,332 @@ +use crate::{block_hash::calculate_execution_block_hash, metrics, Error}; + +use crate::versioned_hashes::verify_versioned_hashes; +use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; +use superstruct::superstruct; +use types::{ + BeaconBlockRef, BeaconStateError, EthSpec, ExecutionBlockHash, ExecutionPayload, + ExecutionPayloadRef, Hash256, VersionedHash, +}; +use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge}; + +#[superstruct( + variants(Merge, Capella, Deneb), + variant_attributes(derive(Clone, Debug, PartialEq),), + map_into(ExecutionPayload), + map_ref_into(ExecutionPayloadRef), + cast_error( + ty = "BeaconStateError", + expr = "BeaconStateError::IncorrectStateVariant" + ), + partial_getter_error( + ty = "BeaconStateError", + expr = "BeaconStateError::IncorrectStateVariant" + ) +)] +#[derive(Clone, Debug, PartialEq)] +pub struct NewPayloadRequest<'block, E: EthSpec> { + #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] + pub execution_payload: &'block ExecutionPayloadMerge, + #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] + pub execution_payload: &'block ExecutionPayloadCapella, + #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] + pub execution_payload: &'block ExecutionPayloadDeneb, + #[superstruct(only(Deneb))] + pub versioned_hashes: Vec, + #[superstruct(only(Deneb))] + pub parent_beacon_block_root: Hash256, +} + +impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { + pub fn parent_hash(&self) -> ExecutionBlockHash { + match self { + Self::Merge(payload) => payload.execution_payload.parent_hash, + Self::Capella(payload) => payload.execution_payload.parent_hash, + Self::Deneb(payload) => payload.execution_payload.parent_hash, + } + } + + pub fn block_hash(&self) -> ExecutionBlockHash { + match self { + Self::Merge(payload) => payload.execution_payload.block_hash, + Self::Capella(payload) => payload.execution_payload.block_hash, + Self::Deneb(payload) => payload.execution_payload.block_hash, + } + } + + pub fn block_number(&self) -> u64 { + match self { + Self::Merge(payload) => payload.execution_payload.block_number, + Self::Capella(payload) => payload.execution_payload.block_number, + Self::Deneb(payload) => payload.execution_payload.block_number, + } + } + + pub fn execution_payload_ref(&self) -> ExecutionPayloadRef<'block, E> { + match self { + Self::Merge(request) => ExecutionPayloadRef::Merge(request.execution_payload), + Self::Capella(request) => ExecutionPayloadRef::Capella(request.execution_payload), + Self::Deneb(request) => ExecutionPayloadRef::Deneb(request.execution_payload), + } + } + + pub fn into_execution_payload(self) -> ExecutionPayload { + match self { + Self::Merge(request) => ExecutionPayload::Merge(request.execution_payload.clone()), + Self::Capella(request) => ExecutionPayload::Capella(request.execution_payload.clone()), + Self::Deneb(request) => ExecutionPayload::Deneb(request.execution_payload.clone()), + } + } + + /// Performs the required verifications of the payload when the chain is optimistically syncing. + /// + /// ## Specification + /// + /// Performs the verifications in the `verify_and_notify_new_payload` function: + /// + /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#modified-verify_and_notify_new_payload + pub fn perform_optimistic_sync_verifications(&self) -> Result<(), Error> { + self.verfiy_payload_block_hash()?; + self.verify_versioned_hashes()?; + + Ok(()) + } + + /// Verify the block hash is consistent locally within Lighthouse. + /// + /// ## Specification + /// + /// Equivalent to `is_valid_block_hash` in the spec: + /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#is_valid_block_hash + pub fn verfiy_payload_block_hash(&self) -> Result<(), Error> { + let payload = self.execution_payload_ref(); + let parent_beacon_block_root = self.parent_beacon_block_root().ok().cloned(); + + let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_VERIFY_BLOCK_HASH); + + let (header_hash, rlp_transactions_root) = + calculate_execution_block_hash(payload, parent_beacon_block_root); + + if header_hash != self.block_hash() { + return Err(Error::BlockHashMismatch { + computed: header_hash, + payload: payload.block_hash(), + transactions_root: rlp_transactions_root, + }); + } + + Ok(()) + } + + /// Verify the versioned hashes computed by the blob transactions match the versioned hashes computed from the commitments. + /// + /// ## Specification + /// + /// Equivalent to `is_valid_versioned_hashes` in the spec: + /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#is_valid_versioned_hashes + pub fn verify_versioned_hashes(&self) -> Result<(), Error> { + if let Ok(versioned_hashes) = self.versioned_hashes() { + verify_versioned_hashes(self.execution_payload_ref(), versioned_hashes) + .map_err(Error::VerifyingVersionedHashes)?; + } + Ok(()) + } +} + +impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<'a, E> { + type Error = BeaconStateError; + + fn try_from(block: BeaconBlockRef<'a, E>) -> Result { + match block { + BeaconBlockRef::Base(_) | BeaconBlockRef::Altair(_) => { + Err(Self::Error::IncorrectStateVariant) + } + BeaconBlockRef::Merge(block_ref) => Ok(Self::Merge(NewPayloadRequestMerge { + execution_payload: &block_ref.body.execution_payload.execution_payload, + })), + BeaconBlockRef::Capella(block_ref) => Ok(Self::Capella(NewPayloadRequestCapella { + execution_payload: &block_ref.body.execution_payload.execution_payload, + })), + BeaconBlockRef::Deneb(block_ref) => Ok(Self::Deneb(NewPayloadRequestDeneb { + execution_payload: &block_ref.body.execution_payload.execution_payload, + versioned_hashes: block_ref + .body + .blob_kzg_commitments + .iter() + .map(kzg_commitment_to_versioned_hash) + .collect(), + parent_beacon_block_root: block_ref.parent_root, + })), + } + } +} + +impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<'a, E> { + type Error = BeaconStateError; + + fn try_from(payload: ExecutionPayloadRef<'a, E>) -> Result { + match payload { + ExecutionPayloadRef::Merge(payload) => Ok(Self::Merge(NewPayloadRequestMerge { + execution_payload: payload, + })), + ExecutionPayloadRef::Capella(payload) => Ok(Self::Capella(NewPayloadRequestCapella { + execution_payload: payload, + })), + ExecutionPayloadRef::Deneb(_) => Err(Self::Error::IncorrectStateVariant), + } + } +} + +#[cfg(test)] +mod test { + use crate::versioned_hashes::Error as VersionedHashError; + use crate::{Error, NewPayloadRequest}; + use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; + use types::{BeaconBlock, ExecPayload, ExecutionBlockHash, Hash256, MainnetEthSpec}; + + #[test] + fn test_optimistic_sync_verifications_valid_block() { + let beacon_block = get_valid_beacon_block(); + let new_payload_request = NewPayloadRequest::try_from(beacon_block.to_ref()) + .expect("should create new payload request"); + + assert!( + new_payload_request + .perform_optimistic_sync_verifications() + .is_ok(), + "validations should pass" + ); + } + + #[test] + fn test_optimistic_sync_verifications_bad_block_hash() { + let mut beacon_block = get_valid_beacon_block(); + let correct_block_hash = beacon_block + .body() + .execution_payload() + .expect("should get payload") + .block_hash(); + let invalid_block_hash = ExecutionBlockHash(Hash256::repeat_byte(0x42)); + + // now mutate the block hash + beacon_block + .body_mut() + .execution_payload_deneb_mut() + .expect("should get payload") + .execution_payload + .block_hash = invalid_block_hash; + + let new_payload_request = NewPayloadRequest::try_from(beacon_block.to_ref()) + .expect("should create new payload request"); + let verification_result = new_payload_request.perform_optimistic_sync_verifications(); + println!("verification_result: {:?}", verification_result); + let got_expected_result = match verification_result { + Err(Error::BlockHashMismatch { + computed, payload, .. + }) => computed == correct_block_hash && payload == invalid_block_hash, + _ => false, + }; + assert!(got_expected_result, "should return expected error"); + } + + #[test] + fn test_optimistic_sync_verifications_bad_versioned_hashes() { + let mut beacon_block = get_valid_beacon_block(); + + let mut commitments: Vec<_> = beacon_block + .body() + .blob_kzg_commitments() + .expect("should get commitments") + .clone() + .into(); + + let correct_versioned_hash = kzg_commitment_to_versioned_hash( + commitments.last().expect("should get last commitment"), + ); + + // mutate the last commitment + commitments + .last_mut() + .expect("should get last commitment") + .0[0] = 0x42; + + // calculate versioned hash from mutated commitment + let bad_versioned_hash = kzg_commitment_to_versioned_hash( + commitments.last().expect("should get last commitment"), + ); + + *beacon_block + .body_mut() + .blob_kzg_commitments_mut() + .expect("should get commitments") = commitments.into(); + + let new_payload_request = NewPayloadRequest::try_from(beacon_block.to_ref()) + .expect("should create new payload request"); + let verification_result = new_payload_request.perform_optimistic_sync_verifications(); + println!("verification_result: {:?}", verification_result); + + let got_expected_result = match verification_result { + Err(Error::VerifyingVersionedHashes(VersionedHashError::VersionHashMismatch { + expected, + found, + })) => expected == bad_versioned_hash && found == correct_versioned_hash, + _ => false, + }; + assert!(got_expected_result, "should return expected error"); + } + + fn get_valid_beacon_block() -> BeaconBlock { + BeaconBlock::Deneb(serde_json::from_str(r#"{ + "slot": "88160", + "proposer_index": "583", + "parent_root": "0x60770cd86a497ca3aa2e91f1687aa3ebafac87af52c30a920b5f40bd9e930eb6", + "state_root": "0x4a0e0abbcbcf576f2cb7387c4289ab13b8a128e32127642f056143d6164941a6", + "body": { + "randao_reveal": "0xb5253d5739496abc4f67c7c92e39e46cca452c2fdfc5275e3e0426a012aa62df82f47f7dece348e28db4bb212f0e793d187120bbd47b8031ed79344116eb4128f0ce0b05ba18cd615bb13966c1bd7d89e23cc769c8e4d8e4a63755f623ac3bed", + "eth1_data": { + "deposit_root": "0xe4785ac914d8673797f886e3151ce2647f81ae070c7ddb6845e65fd1c47d1222", + "deposit_count": "1181", + "block_hash": "0x010671bdfbfce6b0071984a06a7ded6deef13b4f8fdbae402c606a7a0c8780d1" + }, + "graffiti": "0x6c6f6465737461722f6765746800000000000000000000000000000000000000", + "proposer_slashings": [], + "attester_slashings": [], + "attestations": [], + "deposits": [], + "voluntary_exits": [], + "sync_aggregate": { + "sync_committee_bits": "0xfebffffffebfff7fff7f7fffbbefffff6affffffffbfffffefffebfffdbf77fff7fd77ffffefffdff7ffffeffffffe7e5ffffffdefffff7ffbffff7fffffffff", + "sync_committee_signature": "0x91939b5baf2a6f52d405b6dd396f5346ec435eca7d25912c91cc6a2f7030d870d68bebe4f2b21872a06929ff4cf3e5e9191053cb43eb24ebe34b9a75fb88a3acd06baf329c87f68bd664b49891260c698d7bca0f5365870b5b2b3a76f582156c" + }, + "execution_payload": { + "parent_hash": "0xa6f3ed782a992f79ad38da2af91b3e8923c71b801c50bc9033bb35a2e1da885f", + "fee_recipient": "0xf97e180c050e5ab072211ad2c213eb5aee4df134", + "state_root": "0x3bfd1a7f309ed35048c349a8daf01815bdc09a6d5df86ea77d1056f248ba2017", + "receipts_root": "0xcb5b8ffea57cd0fa87194d49bc8bb7fad08c93c9934b886489503c328d15fd36", + "logs_bloom": "0x002000000000000000000000800000000000000000001040000000000000000000000001000000000000000000000000000000000000100000000020000c0800000000000000008000000008000000200000800000000000000000000000000000000000000000008000000000008000000000000000000002000010000000000000000000000000000000000000000000000000000000080000004000000000800000000000000000000100000000000000000000000000000000000800000000000102000000000000000000000000000000080000001000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "prev_randao": "0xb2693020177d99ffbd4c267023be172d759e7306ff51b0e7d677d3148fbd7f1d", + "block_number": "74807", + "gas_limit": "30000000", + "gas_used": "128393", + "timestamp": "1697039520", + "extra_data": "0xd883010d03846765746888676f312e32312e31856c696e7578", + "base_fee_per_gas": "7", + "block_hash": "0xc64f3a43c64aeb98518a237f6279fa03095b9f95ca673c860ad7f16fb9340062", + "transactions": [ + "0x02f9017a8501a1f0ff4382317585012a05f2008512a05f2000830249f094c1b0bc605e2c808aa0867bfc98e51a1fe3e9867f80b901040cc7326300000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000036e534e16b8920d000000000000000000000000fb3e9c7cb92443931ee6b5b9728598d4eb9618c1000000000000000000000000fc7360b3b28cf4204268a8354dbec60720d155d2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000009a054a063f0fe7b9c68de8df91aaa5e96c15ab540000000000000000000000000c8d41b8fcc066cdabaf074d78e5153e8ce018a9c080a07dd9be0d014ffcd5b6883d0917c66b74ba51f0d976c8fc5674af192af6fa9450a02dad2c660974c125f5f22b1e6e8862a292e08cc2b4cafda35af650ee62868a43", + "0x03f8db8501a1f0ff430d84773594008504a817c8008252089454e594b6de0aa4b0188cd1549dd7ba715a455d078080c08504a817c800f863a001253ce00f525e3495cffa0b865eadb90a4c5ee812185cc796af74b6ec0a5dd7a0010720372a4d7dcab84413ed0cfc164fb91fb6ef1562ec2f7a82e912a1d9e129a0015a73e97950397896ed2c47dcab7c0360220bcfb413a8f210a7b6e6264e698880a04402cb0f13c17ef41dca106b1e1520c7aadcbe62984d81171e29914f587d67c1a02db62a8edb581917958e4a3884e7eececbaec114c5ee496e238033e896f997ac" + ], + "withdrawals": [], + "blob_gas_used": "393216", + "excess_blob_gas": "58720256" + }, + "bls_to_execution_changes": [], + "blob_kzg_commitments": [ + "0xa7accb7a25224a8c2e0cee9cd569fc1798665bfbfe780e08945fa9098ec61da4061f5b04e750a88d3340a801850a54fa", + "0xac7b47f99836510ae9076dc5f5da1f370679dea1d47073307a14cbb125cdc7822ae619637135777cb40e13d897fd00a7", + "0x997794110b9655833a88ad5a4ec40a3dc7964877bfbeb04ca1abe1d51bdc43e20e4c5757028896d298d7da954a6f14a1" + ] + } + }"#).expect("should decode")) + } +} diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 868d8194466..664ceabb6cd 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -61,6 +61,7 @@ mod metrics; pub mod payload_cache; mod payload_status; pub mod test_utils; +mod versioned_hashes; /// Indicates the default jwt authenticated execution endpoint. pub const DEFAULT_EXECUTION_ENDPOINT: &str = "http://localhost:8551/"; @@ -141,6 +142,7 @@ pub enum Error { InvalidBlobConversion(String), BeaconStateError(BeaconStateError), PayloadTypeMismatch, + VerifyingVersionedHashes(versioned_hashes::Error), } impl From for Error { @@ -1321,7 +1323,7 @@ impl ExecutionLayer { /// Maps to the `engine_newPayload` JSON-RPC call. pub async fn notify_new_payload( &self, - new_payload_request: NewPayloadRequest, + new_payload_request: NewPayloadRequest<'_, T>, ) -> Result { let _timer = metrics::start_timer_vec( &metrics::EXECUTION_LAYER_REQUEST_TIMES, diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 182cad50faf..6af988fa88f 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -699,7 +699,7 @@ pub fn generate_blobs( Ok((bundle, transactions.into())) } -fn static_valid_tx() -> Result, String> { +pub fn static_valid_tx() -> Result, String> { // This is a real transaction hex encoded, but we don't care about the contents of the transaction. let transaction: EthersTransaction = serde_json::from_str( r#"{ diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 7afeafc321e..77c2410ab1d 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -244,7 +244,7 @@ impl MockExecutionLayer { // TODO: again consider forks let status = self .el - .notify_new_payload(payload.try_into().unwrap()) + .notify_new_payload(payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index b509494703b..425329a520a 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -25,8 +25,8 @@ use warp::{http::StatusCode, Filter, Rejection}; use crate::EngineCapabilities; pub use execution_block_generator::{ - generate_blobs, generate_genesis_block, generate_genesis_header, generate_pow_block, Block, - ExecutionBlockGenerator, + generate_blobs, generate_genesis_block, generate_genesis_header, generate_pow_block, + static_valid_tx, Block, ExecutionBlockGenerator, }; pub use hook::Hook; pub use mock_builder::{MockBuilder, Operation}; diff --git a/beacon_node/execution_layer/src/versioned_hashes.rs b/beacon_node/execution_layer/src/versioned_hashes.rs new file mode 100644 index 00000000000..37bd35646d9 --- /dev/null +++ b/beacon_node/execution_layer/src/versioned_hashes.rs @@ -0,0 +1,135 @@ +extern crate alloy_consensus; +extern crate alloy_rlp; +use alloy_consensus::TxEnvelope; +use alloy_rlp::Decodable; +use types::{EthSpec, ExecutionPayloadRef, Hash256, Unsigned, VersionedHash}; + +#[derive(Debug)] +pub enum Error { + DecodingTransaction(String), + LengthMismatch { expected: usize, found: usize }, + VersionHashMismatch { expected: Hash256, found: Hash256 }, +} + +pub fn verify_versioned_hashes( + execution_payload: ExecutionPayloadRef, + expected_versioned_hashes: &[VersionedHash], +) -> Result<(), Error> { + let versioned_hashes = + extract_versioned_hashes_from_transactions::(execution_payload.transactions())?; + if versioned_hashes.len() != expected_versioned_hashes.len() { + return Err(Error::LengthMismatch { + expected: expected_versioned_hashes.len(), + found: versioned_hashes.len(), + }); + } + for (found, expected) in versioned_hashes + .iter() + .zip(expected_versioned_hashes.iter()) + { + if found != expected { + return Err(Error::VersionHashMismatch { + expected: *expected, + found: *found, + }); + } + } + + Ok(()) +} + +pub fn extract_versioned_hashes_from_transactions( + transactions: &types::Transactions, +) -> Result, Error> { + let mut versioned_hashes = Vec::new(); + + for tx in transactions { + match beacon_tx_to_tx_envelope(tx)? { + TxEnvelope::Eip4844(signed_tx_eip4844) => { + versioned_hashes.extend( + signed_tx_eip4844 + .tx() + .blob_versioned_hashes + .iter() + .map(|fb| Hash256::from(fb.0)), + ); + } + // enumerating all variants explicitly to make pattern irrefutable + // in case new types are added in the future which also have blobs + TxEnvelope::Legacy(_) + | TxEnvelope::TaggedLegacy(_) + | TxEnvelope::Eip2930(_) + | TxEnvelope::Eip1559(_) => {} + } + } + + Ok(versioned_hashes) +} + +pub fn beacon_tx_to_tx_envelope( + tx: &types::Transaction, +) -> Result { + let tx_bytes = Vec::from(tx.clone()); + TxEnvelope::decode(&mut tx_bytes.as_slice()) + .map_err(|e| Error::DecodingTransaction(e.to_string())) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::test_utils::static_valid_tx; + use alloy_consensus::{TxKind, TxLegacy}; + + type E = types::MainnetEthSpec; + + #[test] + fn test_decode_static_transaction() { + let valid_tx = static_valid_tx::().expect("should give me known valid transaction"); + let tx_envelope = beacon_tx_to_tx_envelope(&valid_tx).expect("should decode tx"); + let TxEnvelope::Legacy(signed_tx) = tx_envelope else { + panic!("should decode to legacy transaction"); + }; + + assert!(matches!( + signed_tx.tx(), + TxLegacy { + chain_id: Some(0x01), + nonce: 0x15, + gas_price: 0x4a817c800, + to: TxKind::Call(..), + .. + } + )); + } + + #[test] + fn test_extract_versioned_hashes() { + use serde::Deserialize; + + #[derive(Deserialize)] + #[serde(transparent)] + struct TestTransactions( + #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] types::Transactions, + ); + + let TestTransactions(raw_transactions): TestTransactions = serde_json::from_str(r#"[ + "0x03f901388501a1f0ff430f843b9aca00843b9aca0082520894e7249813d8ccf6fa95a2203f46a64166073d58878080c002f8c6a0012e98362c814f1724262c0d211a1463418a5f6382a8d457b37a2698afbe7b5ea00100ef985761395dfa8ed5ce91f3f2180b612401909e4cb8f33b90c8a454d9baa0013d45411623b90d90f916e4025ada74b453dd4ca093c017c838367c9de0f801a001753e2af0b1e70e7ef80541355b2a035cc9b2c177418bb2a4402a9b346cf84da0011789b520a8068094a92aa0b04db8d8ef1c6c9818947c5210821732b8744049a0011c4c4f95597305daa5f62bf5f690e37fa11f5de05a95d05cac4e2119e394db80a0ccd86a742af0e042d08cbb35d910ddc24bbc6538f9e53be6620d4b6e1bb77662a01a8bacbc614940ac2f5c23ffc00a122c9f085046883de65c88ab0edb859acb99", + "0x02f9017a8501a1f0ff4382363485012a05f2008512a05f2000830249f094c1b0bc605e2c808aa0867bfc98e51a1fe3e9867f80b901040cc7326300000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000009445a285baa43e00000000000000000000000000c500931f24edb821cef6e28f7adb33b38578c82000000000000000000000000fc7360b3b28cf4204268a8354dbec60720d155d2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000009a054a063f0fe7b9c68de8df91aaa5e96c15ab540000000000000000000000000c8d41b8fcc066cdabaf074d78e5153e8ce018a9c080a008e14475c1173cd9f5740c24c08b793f9e16c36c08fa73769db95050e31e3396a019767dcdda26c4a774ca28c9df15d0c20e43bd07bd33ee0f84d6096cb5a1ebed" + ]"#).expect("should get raw transactions"); + let expected_versioned_hashes = vec![ + "0x012e98362c814f1724262c0d211a1463418a5f6382a8d457b37a2698afbe7b5e", + "0x0100ef985761395dfa8ed5ce91f3f2180b612401909e4cb8f33b90c8a454d9ba", + "0x013d45411623b90d90f916e4025ada74b453dd4ca093c017c838367c9de0f801", + "0x01753e2af0b1e70e7ef80541355b2a035cc9b2c177418bb2a4402a9b346cf84d", + "0x011789b520a8068094a92aa0b04db8d8ef1c6c9818947c5210821732b8744049", + "0x011c4c4f95597305daa5f62bf5f690e37fa11f5de05a95d05cac4e2119e394db", + ] + .into_iter() + .map(|tx| Hash256::from_slice(&hex::decode(&tx[2..]).expect("should decode hex"))) + .collect::>(); + + let versioned_hashes = extract_versioned_hashes_from_transactions::(&raw_transactions) + .expect("should get versioned hashes"); + assert_eq!(versioned_hashes, expected_versioned_hashes); + } +} diff --git a/lcli/Dockerfile b/lcli/Dockerfile index aed3628cf32..4f5c3f2972f 100644 --- a/lcli/Dockerfile +++ b/lcli/Dockerfile @@ -1,7 +1,7 @@ # `lcli` requires the full project to be in scope, so this should be built either: # - from the `lighthouse` dir with the command: `docker build -f ./lcli/Dockerflie .` # - from the current directory with the command: `docker build -f ./Dockerfile ../` -FROM rust:1.73.0-bullseye AS builder +FROM rust:1.75.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 8517c66c386..b14227e7edf 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -4,7 +4,7 @@ version = "4.6.0" authors = ["Sigma Prime "] edition = { workspace = true } autotests = false -rust-version = "1.73.0" +rust-version = "1.75.0" [features] default = ["slasher-lmdb"] diff --git a/testing/execution_engine_integration/src/test_rig.rs b/testing/execution_engine_integration/src/test_rig.rs index b0701e80a1b..bfa56f63c0d 100644 --- a/testing/execution_engine_integration/src/test_rig.rs +++ b/testing/execution_engine_integration/src/test_rig.rs @@ -381,7 +381,7 @@ impl TestRig { let status = self .ee_a .execution_layer - .notify_new_payload(valid_payload.clone().try_into().unwrap()) + .notify_new_payload(valid_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -435,7 +435,7 @@ impl TestRig { let status = self .ee_a .execution_layer - .notify_new_payload(invalid_payload.try_into().unwrap()) + .notify_new_payload(invalid_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert!(matches!( @@ -507,7 +507,7 @@ impl TestRig { let status = self .ee_a .execution_layer - .notify_new_payload(second_payload.clone().try_into().unwrap()) + .notify_new_payload(second_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -559,7 +559,7 @@ impl TestRig { let status = self .ee_b .execution_layer - .notify_new_payload(second_payload.clone().try_into().unwrap()) + .notify_new_payload(second_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert!(matches!(status, PayloadStatus::Syncing)); @@ -597,7 +597,7 @@ impl TestRig { let status = self .ee_b .execution_layer - .notify_new_payload(valid_payload.clone().try_into().unwrap()) + .notify_new_payload(valid_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -611,7 +611,7 @@ impl TestRig { let status = self .ee_b .execution_layer - .notify_new_payload(second_payload.clone().try_into().unwrap()) + .notify_new_payload(second_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); From f21472991dbbc9e6e18310a01fc2c4d9ce7c7348 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 18 Feb 2024 21:22:15 -0500 Subject: [PATCH 08/21] check the da cache and the attester cache in responding to RPC requests (#5138) * check the da cache and the attester cache in responding to RPC requests * use the processing cache instead * update comment * add da cache metrics * rename early attester cache method * Merge branch 'unstable' of https://github.com/sigp/lighthouse into check-da-cache-in-rpc-response * make rustup update run on the runners * Revert "make rustup update run on the runners" This reverts commit d097e9bfa84a13b1d7813c03df38e7756fb0bfc5. --- .../beacon_chain/src/beacon_block_streamer.rs | 33 ++++++------ beacon_node/beacon_chain/src/beacon_chain.rs | 30 +++-------- .../beacon_chain/src/block_verification.rs | 17 +++++++ .../src/block_verification_types.rs | 7 +++ .../src/data_availability_checker.rs | 51 +++++++++++++++---- .../availability_view.rs | 20 ++------ .../overflow_lru_cache.rs | 25 +++++++++ .../processing_cache.rs | 14 +++-- .../state_lru_cache.rs | 3 +- beacon_node/beacon_chain/src/metrics.rs | 43 ++++++++++++++++ .../network_beacon_processor/rpc_methods.rs | 2 +- 11 files changed, 171 insertions(+), 74 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index 9312d4511d5..4f4f8ed1fe0 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -19,7 +19,7 @@ use types::{ }; #[derive(PartialEq)] -pub enum CheckEarlyAttesterCache { +pub enum CheckCaches { Yes, No, } @@ -385,14 +385,14 @@ impl EngineRequest { pub struct BeaconBlockStreamer { execution_layer: ExecutionLayer, - check_early_attester_cache: CheckEarlyAttesterCache, + check_caches: CheckCaches, beacon_chain: Arc>, } impl BeaconBlockStreamer { pub fn new( beacon_chain: &Arc>, - check_early_attester_cache: CheckEarlyAttesterCache, + check_caches: CheckCaches, ) -> Result { let execution_layer = beacon_chain .execution_layer @@ -402,17 +402,17 @@ impl BeaconBlockStreamer { Ok(Self { execution_layer, - check_early_attester_cache, + check_caches, beacon_chain: beacon_chain.clone(), }) } - fn check_early_attester_cache( - &self, - root: Hash256, - ) -> Option>> { - if self.check_early_attester_cache == CheckEarlyAttesterCache::Yes { - self.beacon_chain.early_attester_cache.get_block(root) + fn check_caches(&self, root: Hash256) -> Option>> { + if self.check_caches == CheckCaches::Yes { + self.beacon_chain + .data_availability_checker + .get_block(&root) + .or(self.beacon_chain.early_attester_cache.get_block(root)) } else { None } @@ -422,10 +422,7 @@ impl BeaconBlockStreamer { let mut db_blocks = Vec::new(); for root in block_roots { - if let Some(cached_block) = self - .check_early_attester_cache(root) - .map(LoadedBeaconBlock::Full) - { + if let Some(cached_block) = self.check_caches(root).map(LoadedBeaconBlock::Full) { db_blocks.push((root, Ok(Some(cached_block)))); continue; } @@ -554,7 +551,7 @@ impl BeaconBlockStreamer { "Using slower fallback method of eth_getBlockByHash()" ); for root in block_roots { - let cached_block = self.check_early_attester_cache(root); + let cached_block = self.check_caches(root); let block_result = if cached_block.is_some() { Ok(cached_block) } else { @@ -682,7 +679,7 @@ impl From for BeaconChainError { #[cfg(test)] mod tests { - use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache}; + use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckCaches}; use crate::test_utils::{test_spec, BeaconChainHarness, EphemeralHarnessType}; use execution_layer::test_utils::{Block, DEFAULT_ENGINE_CAPABILITIES}; use execution_layer::EngineCapabilities; @@ -804,7 +801,7 @@ mod tests { let start = epoch * slots_per_epoch; let mut epoch_roots = vec![Hash256::zero(); slots_per_epoch]; epoch_roots[..].clone_from_slice(&block_roots[start..(start + slots_per_epoch)]); - let streamer = BeaconBlockStreamer::new(&harness.chain, CheckEarlyAttesterCache::No) + let streamer = BeaconBlockStreamer::new(&harness.chain, CheckCaches::No) .expect("should create streamer"); let (block_tx, mut block_rx) = mpsc::unbounded_channel(); streamer.stream(epoch_roots.clone(), block_tx).await; @@ -945,7 +942,7 @@ mod tests { let start = epoch * slots_per_epoch; let mut epoch_roots = vec![Hash256::zero(); slots_per_epoch]; epoch_roots[..].clone_from_slice(&block_roots[start..(start + slots_per_epoch)]); - let streamer = BeaconBlockStreamer::new(&harness.chain, CheckEarlyAttesterCache::No) + let streamer = BeaconBlockStreamer::new(&harness.chain, CheckCaches::No) .expect("should create streamer"); let (block_tx, mut block_rx) = mpsc::unbounded_channel(); streamer.stream(epoch_roots.clone(), block_tx).await; diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b05ba01ee7e..20a93e31e8d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4,7 +4,7 @@ use crate::attestation_verification::{ VerifiedUnaggregatedAttestation, }; use crate::attester_cache::{AttesterCache, AttesterCacheKey}; -use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache}; +use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckCaches}; use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_verification::{GossipBlobError, GossipVerifiedBlob}; @@ -1131,7 +1131,7 @@ impl BeaconChain { /// ## Errors /// /// May return a database error. - pub fn get_blocks_checking_early_attester_cache( + pub fn get_blocks_checking_caches( self: &Arc, block_roots: Vec, executor: &TaskExecutor, @@ -1144,10 +1144,8 @@ impl BeaconChain { >, Error, > { - Ok( - BeaconBlockStreamer::::new(self, CheckEarlyAttesterCache::Yes)? - .launch_stream(block_roots, executor), - ) + Ok(BeaconBlockStreamer::::new(self, CheckCaches::Yes)? + .launch_stream(block_roots, executor)) } pub fn get_blocks( @@ -1163,10 +1161,8 @@ impl BeaconChain { >, Error, > { - Ok( - BeaconBlockStreamer::::new(self, CheckEarlyAttesterCache::No)? - .launch_stream(block_roots, executor), - ) + Ok(BeaconBlockStreamer::::new(self, CheckCaches::No)? + .launch_stream(block_roots, executor)) } pub fn get_blobs_checking_early_attester_cache( @@ -2960,18 +2956,8 @@ impl BeaconChain { unverified_block: B, notify_execution_layer: NotifyExecutionLayer, ) -> Result> { - if let Ok(commitments) = unverified_block - .block() - .message() - .body() - .blob_kzg_commitments() - { - self.data_availability_checker.notify_block_commitments( - unverified_block.block().slot(), - block_root, - commitments.clone(), - ); - }; + self.data_availability_checker + .notify_block(block_root, unverified_block.block_cloned()); let r = self .process_block(block_root, unverified_block, notify_execution_layer, || { Ok(()) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index e8df5b811ed..ac3d3e3ab80 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -764,6 +764,7 @@ pub trait IntoExecutionPendingBlock: Sized { ) -> Result, BlockSlashInfo>>; fn block(&self) -> &SignedBeaconBlock; + fn block_cloned(&self) -> Arc>; } impl GossipVerifiedBlock { @@ -1017,6 +1018,10 @@ impl IntoExecutionPendingBlock for GossipVerifiedBlock &SignedBeaconBlock { self.block.as_block() } + + fn block_cloned(&self) -> Arc> { + self.block.clone() + } } impl SignatureVerifiedBlock { @@ -1168,6 +1173,10 @@ impl IntoExecutionPendingBlock for SignatureVerifiedBloc fn block(&self) -> &SignedBeaconBlock { self.block.as_block() } + + fn block_cloned(&self) -> Arc> { + self.block.block_cloned() + } } impl IntoExecutionPendingBlock for Arc> { @@ -1198,6 +1207,10 @@ impl IntoExecutionPendingBlock for Arc &SignedBeaconBlock { self } + + fn block_cloned(&self) -> Arc> { + self.clone() + } } impl IntoExecutionPendingBlock for RpcBlock { @@ -1228,6 +1241,10 @@ impl IntoExecutionPendingBlock for RpcBlock fn block(&self) -> &SignedBeaconBlock { self.as_block() } + + fn block_cloned(&self) -> Arc> { + self.block_cloned() + } } impl ExecutionPendingBlock { diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index a6840ed7648..edba7a211cb 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -46,6 +46,13 @@ impl RpcBlock { } } + pub fn block_cloned(&self) -> Arc> { + match &self.block { + RpcBlockInner::Block(block) => block.clone(), + RpcBlockInner::BlockAndBlobs(block, _) => block.clone(), + } + } + pub fn blobs(&self) -> Option<&BlobSidecarList> { match &self.block { RpcBlockInner::Block(_) => None, diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 48d505e9e7b..f906032ecd2 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -20,7 +20,7 @@ use std::fmt::Debug; use std::num::NonZeroUsize; use std::sync::Arc; use task_executor::TaskExecutor; -use types::beacon_block_body::{KzgCommitmentOpts, KzgCommitments}; +use types::beacon_block_body::KzgCommitmentOpts; use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; use types::{BlobSidecarList, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot}; @@ -192,6 +192,14 @@ impl DataAvailabilityChecker { self.availability_cache.peek_blob(blob_id) } + /// Get a block from the availability cache. Includes any blocks we are currently processing. + pub fn get_block(&self, block_root: &Hash256) -> Option>> { + self.processing_cache + .read() + .get(block_root) + .and_then(|cached| cached.block.clone()) + } + /// Put a list of blobs received via RPC into the availability cache. This performs KZG /// verification on the blobs in the list. pub fn put_rpc_blobs( @@ -344,20 +352,16 @@ impl DataAvailabilityChecker { block.num_expected_blobs() > 0 && self.da_check_required_for_epoch(block.epoch()) } - /// Adds block commitments to the processing cache. These commitments are unverified but caching + /// Adds a block to the processing cache. This block's commitments are unverified but caching /// them here is useful to avoid duplicate downloads of blocks, as well as understanding - /// our blob download requirements. - pub fn notify_block_commitments( - &self, - slot: Slot, - block_root: Hash256, - commitments: KzgCommitments, - ) { + /// our blob download requirements. We will also serve this over RPC. + pub fn notify_block(&self, block_root: Hash256, block: Arc>) { + let slot = block.slot(); self.processing_cache .write() .entry(block_root) .or_insert_with(|| ProcessingComponents::new(slot)) - .merge_block(commitments); + .merge_block(block); } /// Add a single blob commitment to the processing cache. This commitment is unverified but caching @@ -450,6 +454,24 @@ impl DataAvailabilityChecker { pub fn persist_all(&self) -> Result<(), AvailabilityCheckError> { self.availability_cache.write_all_to_disk() } + + /// Collects metrics from the data availability checker. + pub fn metrics(&self) -> DataAvailabilityCheckerMetrics { + DataAvailabilityCheckerMetrics { + processing_cache_size: self.processing_cache.read().len(), + num_store_entries: self.availability_cache.num_store_entries(), + state_cache_size: self.availability_cache.state_cache_size(), + block_cache_size: self.availability_cache.block_cache_size(), + } + } +} + +/// Helper struct to group data availability checker metrics. +pub struct DataAvailabilityCheckerMetrics { + pub processing_cache_size: usize, + pub num_store_entries: usize, + pub state_cache_size: usize, + pub block_cache_size: usize, } pub fn start_availability_cache_maintenance_service( @@ -597,6 +619,15 @@ pub enum MaybeAvailableBlock { }, } +impl MaybeAvailableBlock { + pub fn block_cloned(&self) -> Arc> { + match self { + Self::Available(block) => block.block_cloned(), + Self::AvailabilityPending { block, .. } => block.clone(), + } + } +} + #[derive(Debug, Clone)] pub enum MissingBlobs { /// We know for certain these blobs are missing. diff --git a/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs b/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs index 776f81ee545..65093db26bd 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs @@ -182,9 +182,9 @@ macro_rules! impl_availability_view { impl_availability_view!( ProcessingComponents, - KzgCommitments, + Arc>, KzgCommitment, - block_commitments, + block, blob_commitments ); @@ -212,12 +212,6 @@ pub trait GetCommitment { fn get_commitment(&self) -> &KzgCommitment; } -// These implementations are required to implement `AvailabilityView` for `ProcessingView`. -impl GetCommitments for KzgCommitments { - fn get_commitments(&self) -> KzgCommitments { - self.clone() - } -} impl GetCommitment for KzgCommitment { fn get_commitment(&self) -> &KzgCommitment { self @@ -310,7 +304,7 @@ pub mod tests { } type ProcessingViewSetup = ( - KzgCommitments, + Arc>, FixedVector, ::MaxBlobsPerBlock>, FixedVector, ::MaxBlobsPerBlock>, ); @@ -320,12 +314,6 @@ pub mod tests { valid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, invalid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, ) -> ProcessingViewSetup { - let commitments = block - .message() - .body() - .blob_kzg_commitments() - .unwrap() - .clone(); let blobs = FixedVector::from( valid_blobs .iter() @@ -338,7 +326,7 @@ pub mod tests { .map(|blob_opt| blob_opt.as_ref().map(|blob| blob.kzg_commitment)) .collect::>(), ); - (commitments, blobs, invalid_blobs) + (Arc::new(block), blobs, invalid_blobs) } type PendingComponentsSetup = ( diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 34c9bc76f6e..80cbc6c8990 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -363,6 +363,16 @@ impl Critical { } } } + + /// Returns the number of pending component entries in memory. + pub fn num_blocks(&self) -> usize { + self.in_memory.len() + } + + /// Returns the number of entries that have overflowed to disk. + pub fn num_store_entries(&self) -> usize { + self.store_keys.len() + } } /// This is the main struct for this module. Outside methods should @@ -671,6 +681,21 @@ impl OverflowLRUCache { pub fn state_lru_cache(&self) -> &StateLRUCache { &self.state_cache } + + /// Number of states stored in memory in the cache. + pub fn state_cache_size(&self) -> usize { + self.state_cache.lru_cache().read().len() + } + + /// Number of pending component entries in memory in the cache. + pub fn block_cache_size(&self) -> usize { + self.critical.read().num_blocks() + } + + /// Returns the number of entries in the cache that have overflowed to disk. + pub fn num_store_entries(&self) -> usize { + self.critical.read().num_store_entries() + } } impl ssz::Encode for OverflowKey { diff --git a/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs index 969034c6570..af94803dcfb 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs @@ -1,8 +1,9 @@ use crate::data_availability_checker::AvailabilityView; use std::collections::hash_map::Entry; use std::collections::HashMap; -use types::beacon_block_body::{KzgCommitmentOpts, KzgCommitments}; -use types::{EthSpec, Hash256, Slot}; +use std::sync::Arc; +use types::beacon_block_body::KzgCommitmentOpts; +use types::{EthSpec, Hash256, SignedBeaconBlock, Slot}; /// This cache is used only for gossip blocks/blobs and single block/blob lookups, to give req/resp /// a view of what we have and what we require. This cache serves a slightly different purpose than @@ -37,6 +38,9 @@ impl ProcessingCache { } roots_missing_components } + pub fn len(&self) -> usize { + self.processing_cache.len() + } } #[derive(Debug, Clone)] @@ -45,7 +49,7 @@ pub struct ProcessingComponents { /// Blobs required for a block can only be known if we have seen the block. So `Some` here /// means we've seen it, a `None` means we haven't. The `kzg_commitments` value helps us figure /// out whether incoming blobs actually match the block. - pub block_commitments: Option>, + pub block: Option>>, /// `KzgCommitments` for blobs are always known, even if we haven't seen the block. See /// `AvailabilityView`'s trait definition for more details. pub blob_commitments: KzgCommitmentOpts, @@ -55,7 +59,7 @@ impl ProcessingComponents { pub fn new(slot: Slot) -> Self { Self { slot, - block_commitments: None, + block: None, blob_commitments: KzgCommitmentOpts::::default(), } } @@ -67,7 +71,7 @@ impl ProcessingComponents { pub fn empty(_block_root: Hash256) -> Self { Self { slot: Slot::new(0), - block_commitments: None, + block: None, blob_commitments: KzgCommitmentOpts::::default(), } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs index bd125a7f42a..35c114db542 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs @@ -190,8 +190,7 @@ impl StateLRUCache { }) } - /// returns the state cache for inspection in tests - #[cfg(test)] + /// returns the state cache for inspection pub fn lru_cache(&self) -> &RwLock>> { &self.states } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 24c05e01f22..abac2c80e7f 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1129,6 +1129,31 @@ lazy_static! { Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) ); + + /* + * Data Availability cache metrics + */ + pub static ref DATA_AVAILABILITY_PROCESSING_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_processing_cache_size", + "Number of entries in the data availability processing cache." + ); + pub static ref DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_overflow_memory_block_cache_size", + "Number of entries in the data availability overflow block memory cache." + ); + pub static ref DATA_AVAILABILITY_OVERFLOW_MEMORY_STATE_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_overflow_memory_state_cache_size", + "Number of entries in the data availability overflow state memory cache." + ); + pub static ref DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_overflow_store_cache_size", + "Number of entries in the data availability overflow store cache." + ); + /* * light_client server metrics */ @@ -1171,6 +1196,24 @@ pub fn scrape_for_metrics(beacon_chain: &BeaconChain) { ) } + let da_checker_metrics = beacon_chain.data_availability_checker.metrics(); + set_gauge_by_usize( + &DATA_AVAILABILITY_PROCESSING_CACHE_SIZE, + da_checker_metrics.processing_cache_size, + ); + set_gauge_by_usize( + &DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE, + da_checker_metrics.block_cache_size, + ); + set_gauge_by_usize( + &DATA_AVAILABILITY_OVERFLOW_MEMORY_STATE_CACHE_SIZE, + da_checker_metrics.state_cache_size, + ); + set_gauge_by_usize( + &DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE, + da_checker_metrics.num_store_entries, + ); + if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() { set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size); set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_LOOKUP_COUNT, num_lookups); diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index a731dea7c19..66c98ff3b84 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -140,7 +140,7 @@ impl NetworkBeaconProcessor { let requested_blocks = request.block_roots().len(); let mut block_stream = match self .chain - .get_blocks_checking_early_attester_cache(request.block_roots().to_vec(), &executor) + .get_blocks_checking_caches(request.block_roots().to_vec(), &executor) { Ok(block_stream) => block_stream, Err(e) => return error!(self.log, "Error getting block stream"; "error" => ?e), From e22c9eed8f9de26f14d78989d363b20009a20875 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 19 Feb 2024 13:22:19 +1100 Subject: [PATCH 09/21] Add Deneb fork epoch for Gnosis (#5242) * Add Deneb fork epoch for Gnosis * Add deneb constants * Update common/eth2_network_config/built_in_network_configs/gnosis/config.yaml Co-authored-by: realbigsean * Adjust `min_epochs_for_block_requests` * Change `MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT` * Fix chain spec `max_per_epoch_activation_churn_limit` * Fix chain spec values again --- .../built_in_network_configs/gnosis/config.yaml | 14 ++++++++++++-- consensus/types/src/chain_spec.rs | 8 ++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index 18a08ed12d7..2311d6db0f9 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -41,7 +41,7 @@ CAPELLA_FORK_VERSION: 0x03000064 CAPELLA_FORK_EPOCH: 648704 # Deneb DENEB_FORK_VERSION: 0x04000064 -DENEB_FORK_EPOCH: 18446744073709551615 +DENEB_FORK_EPOCH: 889856 # 2024-03-11T18:30:20.000Z # Sharding SHARDING_FORK_VERSION: 0x03000064 SHARDING_FORK_EPOCH: 18446744073709551615 @@ -75,7 +75,7 @@ EJECTION_BALANCE: 16000000000 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**3 (= 8) -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 2 # 2**12 (= 4096) CHURN_LIMIT_QUOTIENT: 4096 @@ -106,3 +106,13 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**14` (= 16384 epochs, ~15 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index c32c67fa330..88f989d2a5a 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -809,7 +809,7 @@ impl ChainSpec { max_committees_per_slot: 64, target_committee_size: 128, min_per_epoch_churn_limit: 4, - max_per_epoch_activation_churn_limit: 8, + max_per_epoch_activation_churn_limit: 2, churn_limit_quotient: 4_096, shuffle_round_count: 90, min_genesis_active_validator_count: 4_096, @@ -944,7 +944,7 @@ impl ChainSpec { * Deneb hard fork params */ deneb_fork_version: [0x04, 0x00, 0x00, 0x64], - deneb_fork_epoch: None, + deneb_fork_epoch: Some(Epoch::new(889856)), /* * Network specific @@ -958,7 +958,7 @@ impl ChainSpec { target_aggregators_per_committee: 16, epochs_per_subnet_subscription: default_epochs_per_subnet_subscription(), gossip_max_size: default_gossip_max_size(), - min_epochs_for_block_requests: default_min_epochs_for_block_requests(), + min_epochs_for_block_requests: 33024, max_chunk_size: default_max_chunk_size(), ttfb_timeout: default_ttfb_timeout(), resp_timeout: default_resp_timeout(), @@ -975,7 +975,7 @@ impl ChainSpec { */ max_request_blocks_deneb: default_max_request_blocks_deneb(), max_request_blob_sidecars: default_max_request_blob_sidecars(), - min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(), + min_epochs_for_blob_sidecars_requests: 16384, blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), /* From c9702cb0a1a0307d2ccf8f8d3d072ef94f3d4c59 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 19 Feb 2024 13:22:23 +1100 Subject: [PATCH 10/21] Download checkpoint blobs during checkpoint sync (#5252) * MVP implementation (untested) * update store checkpoint sync test * update cli help * Merge pull request #5253 from realbigsean/checkpoint-blobs-sean Checkpoint blobs sean * Warn only if blobs are missing from server * Merge remote-tracking branch 'origin/unstable' into checkpoint-blobs * Verify checkpoint blobs * Move blob verification earlier --- Cargo.lock | 1 + beacon_node/beacon_chain/src/builder.rs | 39 +++++++++++++-- beacon_node/beacon_chain/tests/store_tests.rs | 19 ++++++- beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/builder.rs | 49 +++++++++++++++++-- beacon_node/client/src/config.rs | 1 + beacon_node/src/cli.rs | 9 ++++ beacon_node/src/config.rs | 5 +- book/src/help_bn.md | 3 ++ consensus/types/src/beacon_block_body.rs | 6 +++ 10 files changed, 122 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c32fed0874..5885a7387f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1495,6 +1495,7 @@ dependencies = [ "eth1", "eth2", "eth2_config", + "ethereum_ssz", "execution_layer", "futures", "genesis", diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index abd0e6b6a82..c75c3f695b3 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -39,8 +39,8 @@ use std::time::Duration; use store::{Error as StoreError, HotColdDB, ItemStore, KeyValueStoreOp}; use task_executor::{ShutdownReason, TaskExecutor}; use types::{ - BeaconBlock, BeaconState, ChainSpec, Checkpoint, Epoch, EthSpec, Graffiti, Hash256, Signature, - SignedBeaconBlock, Slot, + BeaconBlock, BeaconState, BlobSidecarList, ChainSpec, Checkpoint, Epoch, EthSpec, Graffiti, + Hash256, Signature, SignedBeaconBlock, Slot, }; /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing @@ -432,6 +432,7 @@ where mut self, mut weak_subj_state: BeaconState, weak_subj_block: SignedBeaconBlock, + weak_subj_blobs: Option>, genesis_state: BeaconState, ) -> Result { let store = self @@ -490,6 +491,29 @@ where )); } + // Verify that blobs (if provided) match the block. + if let Some(blobs) = &weak_subj_blobs { + let commitments = weak_subj_block + .message() + .body() + .blob_kzg_commitments() + .map_err(|e| format!("Blobs provided but block does not reference them: {e:?}"))?; + if blobs.len() != commitments.len() { + return Err(format!( + "Wrong number of blobs, expected: {}, got: {}", + commitments.len(), + blobs.len() + )); + } + if commitments + .iter() + .zip(blobs.iter()) + .any(|(commitment, blob)| *commitment != blob.kzg_commitment) + { + return Err("Checkpoint blob does not match block commitment".into()); + } + } + // Set the store's split point *before* storing genesis so that genesis is stored // immediately in the freezer DB. store.set_split(weak_subj_slot, weak_subj_state_root, weak_subj_block_root); @@ -511,14 +535,19 @@ where .do_atomically(block_root_batch) .map_err(|e| format!("Error writing frozen block roots: {e:?}"))?; - // Write the state and block non-atomically, it doesn't matter if they're forgotten + // Write the state, block and blobs non-atomically, it doesn't matter if they're forgotten // about on a crash restart. store .put_state(&weak_subj_state_root, &weak_subj_state) - .map_err(|e| format!("Failed to store weak subjectivity state: {:?}", e))?; + .map_err(|e| format!("Failed to store weak subjectivity state: {e:?}"))?; store .put_block(&weak_subj_block_root, weak_subj_block.clone()) - .map_err(|e| format!("Failed to store weak subjectivity block: {:?}", e))?; + .map_err(|e| format!("Failed to store weak subjectivity block: {e:?}"))?; + if let Some(blobs) = weak_subj_blobs { + store + .put_blobs(&weak_subj_block_root, blobs) + .map_err(|e| format!("Failed to store weak subjectivity blobs: {e:?}"))?; + } // Stage the database's metadata fields for atomic storage when `build` is called. // This prevents the database from restarting in an inconsistent state if the anchor diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index ffd1b843a18..ff201729821 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2396,6 +2396,7 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .get_full_block(&wss_block_root) .unwrap() .unwrap(); + let wss_blobs_opt = harness.chain.store.get_blobs(&wss_block_root).unwrap(); let wss_state = full_store .get_state(&wss_state_root, Some(checkpoint_slot)) .unwrap() @@ -2438,7 +2439,12 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .custom_spec(test_spec::()) .task_executor(harness.chain.task_executor.clone()) .logger(log.clone()) - .weak_subjectivity_state(wss_state, wss_block.clone(), genesis_state) + .weak_subjectivity_state( + wss_state, + wss_block.clone(), + wss_blobs_opt.clone(), + genesis_state, + ) .unwrap() .store_migrator_config(MigratorConfig::default().blocking()) .dummy_eth1_backend() @@ -2456,6 +2462,17 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .expect("should build"); let beacon_chain = Arc::new(beacon_chain); + let wss_block_root = wss_block.canonical_root(); + let store_wss_block = harness + .chain + .get_block(&wss_block_root) + .await + .unwrap() + .unwrap(); + let store_wss_blobs_opt = beacon_chain.store.get_blobs(&wss_block_root).unwrap(); + + assert_eq!(store_wss_block, wss_block); + assert_eq!(store_wss_blobs_opt, wss_blobs_opt); // Apply blocks forward to reach head. let chain_dump = harness.chain.chain_dump().unwrap(); diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 0160cad4b20..03cbcc9ff7f 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -45,3 +45,4 @@ monitoring_api = { workspace = true } execution_layer = { workspace = true } beacon_processor = { workspace = true } num_cpus = { workspace = true } +ethereum_ssz = { workspace = true } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index de6e08cc37a..558e5cbc84f 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -36,6 +36,7 @@ use network::{NetworkConfig, NetworkSenders, NetworkService}; use slasher::Slasher; use slasher_service::SlasherService; use slog::{debug, info, warn, Logger}; +use ssz::Decode; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -44,7 +45,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use timer::spawn_timer; use tokio::sync::oneshot; use types::{ - test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec, + test_utils::generate_deterministic_keypairs, BeaconState, BlobSidecarList, ChainSpec, EthSpec, ExecutionBlockHash, Hash256, SignedBeaconBlock, }; @@ -319,6 +320,7 @@ where ClientGenesis::WeakSubjSszBytes { anchor_state_bytes, anchor_block_bytes, + anchor_blobs_bytes, } => { info!(context.log(), "Starting checkpoint sync"); if config.chain.genesis_backfill { @@ -332,10 +334,25 @@ where .map_err(|e| format!("Unable to parse weak subj state SSZ: {:?}", e))?; let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec) .map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?; + let anchor_blobs = if anchor_block.message().body().has_blobs() { + let anchor_blobs_bytes = anchor_blobs_bytes + .ok_or("Blobs for checkpoint must be provided using --checkpoint-blobs")?; + Some( + BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes) + .map_err(|e| format!("Unable to parse weak subj blobs SSZ: {e:?}"))?, + ) + } else { + None + }; let genesis_state = genesis_state(&runtime_context, &config, log).await?; builder - .weak_subjectivity_state(anchor_state, anchor_block, genesis_state) + .weak_subjectivity_state( + anchor_state, + anchor_block, + anchor_blobs, + genesis_state, + ) .map(|v| (v, None))? } ClientGenesis::CheckpointSyncUrl { url } => { @@ -430,9 +447,33 @@ where e => format!("Error fetching finalized block from remote: {:?}", e), })? .ok_or("Finalized block missing from remote, it returned 404")?; + let block_root = block.canonical_root(); debug!(context.log(), "Downloaded finalized block"); + let blobs = if block.message().body().has_blobs() { + debug!(context.log(), "Downloading finalized blobs"); + if let Some(response) = remote + .get_blobs::(BlockId::Root(block_root), None) + .await + .map_err(|e| format!("Error fetching finalized blobs from remote: {e:?}"))? + { + debug!(context.log(), "Downloaded finalized blobs"); + Some(response.data) + } else { + warn!( + context.log(), + "Checkpoint server is missing blobs"; + "block_root" => %block_root, + "hint" => "use a different URL or ask the provider to update", + "impact" => "db will be slightly corrupt until these blobs are pruned", + ); + None + } + } else { + None + }; + let genesis_state = genesis_state(&runtime_context, &config, log).await?; info!( @@ -440,7 +481,7 @@ where "Loaded checkpoint block and state"; "block_slot" => block.slot(), "state_slot" => state.slot(), - "block_root" => ?block.canonical_root(), + "block_root" => ?block_root, ); let service = @@ -468,7 +509,7 @@ where }); builder - .weak_subjectivity_state(state, block, genesis_state) + .weak_subjectivity_state(state, block, blobs, genesis_state) .map(|v| (v, service))? } ClientGenesis::DepositContract => { diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 275f9998640..197f21c64ed 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -35,6 +35,7 @@ pub enum ClientGenesis { WeakSubjSszBytes { anchor_state_bytes: Vec, anchor_block_bytes: Vec, + anchor_blobs_bytes: Option>, }, CheckpointSyncUrl { url: SensitiveUrl, diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 4ed60d27b0f..fa4edc34d22 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -939,6 +939,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .requires("checkpoint-state") ) + .arg( + Arg::with_name("checkpoint-blobs") + .long("checkpoint-blobs") + .help("Set the checkpoint blobs to start syncing from. Must be aligned and match \ + --checkpoint-block. Using --checkpoint-sync-url instead is recommended.") + .value_name("BLOBS_SSZ") + .takes_value(true) + .requires("checkpoint-block") + ) .arg( Arg::with_name("checkpoint-sync-url") .long("checkpoint-sync-url") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 1699c51784f..fc6132f8c9b 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -512,9 +512,10 @@ pub fn get_config( client_config.genesis = if eth2_network_config.genesis_state_is_known() { // Set up weak subjectivity sync, or start from the hardcoded genesis state. - if let (Some(initial_state_path), Some(initial_block_path)) = ( + if let (Some(initial_state_path), Some(initial_block_path), opt_initial_blobs_path) = ( cli_args.value_of("checkpoint-state"), cli_args.value_of("checkpoint-block"), + cli_args.value_of("checkpoint-blobs"), ) { let read = |path: &str| { use std::fs::File; @@ -530,10 +531,12 @@ pub fn get_config( let anchor_state_bytes = read(initial_state_path)?; let anchor_block_bytes = read(initial_block_path)?; + let anchor_blobs_bytes = opt_initial_blobs_path.map(read).transpose()?; ClientGenesis::WeakSubjSszBytes { anchor_state_bytes, anchor_block_bytes, + anchor_blobs_bytes, } } else if let Some(remote_bn_url) = cli_args.value_of("checkpoint-sync-url") { let url = SensitiveUrl::parse(remote_bn_url) diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 3d2124964b2..b2a922020f5 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -179,6 +179,9 @@ OPTIONS: --builder-user-agent The HTTP user agent to send alongside requests to the builder URL. The default is Lighthouse's version string. + --checkpoint-blobs + Set the checkpoint blobs to start syncing from. Must be aligned and match --checkpoint-block. Using + --checkpoint-sync-url instead is recommended. --checkpoint-block Set a checkpoint block to start syncing from. Must be aligned and match --checkpoint-state. Using --checkpoint-sync-url instead is recommended. diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 146dff895c8..b0a942d74a4 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -176,6 +176,12 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, } } } + + /// Return `true` if this block body has a non-zero number of blobs. + pub fn has_blobs(self) -> bool { + self.blob_kzg_commitments() + .map_or(false, |blobs| !blobs.is_empty()) + } } impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, Payload> { From 4d625951b823f7365dcaeb4e5c8f15263af43eb0 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 19 Feb 2024 07:17:58 +0200 Subject: [PATCH 11/21] Deprecate env_log flag in tracing layer (#5228) * deprecate terminal logs file in tracing layer * sink writer --- common/logging/src/lib.rs | 8 ++------ lighthouse/src/main.rs | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index caf3e1d2fb2..3a5a5209b02 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -223,7 +223,7 @@ impl TimeLatch { } } -pub fn create_tracing_layer(base_tracing_log_path: PathBuf, turn_on_terminal_logs: bool) { +pub fn create_tracing_layer(base_tracing_log_path: PathBuf) { let filter_layer = match tracing_subscriber::EnvFilter::try_from_default_env() .or_else(|_| tracing_subscriber::EnvFilter::try_new("warn")) { @@ -268,11 +268,7 @@ pub fn create_tracing_layer(base_tracing_log_path: PathBuf, turn_on_terminal_log if let Err(e) = tracing_subscriber::fmt() .with_env_filter(filter_layer) - .with_writer(move || { - tracing_subscriber::fmt::writer::OptionalWriter::::from( - turn_on_terminal_logs.then(std::io::stdout), - ) - }) + .with_writer(std::io::sink) .finish() .with(MetricsLayer) .with(custom_layer) diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 022b208359d..d646b9764cd 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -556,9 +556,7 @@ fn run( let path = tracing_log_path.clone().unwrap(); - let turn_on_terminal_logs = matches.is_present("env_log"); - - logging::create_tracing_layer(path, turn_on_terminal_logs); + logging::create_tracing_layer(path); // Allow Prometheus to export the time at which the process was started. metrics::expose_process_start_time(&log); From a229b5272354cee313c6267456f724f3b9b92ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 19 Feb 2024 07:16:01 +0000 Subject: [PATCH 12/21] Deactivate RPC Connection Handler after goodbye message is sent (#5250) * Deactivate RPC Connection Handler after goodbye message is sent * nit: use to_string instead of format * Merge branch 'unstable' of https://github.com/sigp/lighthouse into rpc-shutdown-improvement * clippy * Fix cargo.lock * Merge latest unstable --- Cargo.lock | 376 +++++++++--------- .../lighthouse_network/src/rpc/handler.rs | 32 +- beacon_node/lighthouse_network/src/rpc/mod.rs | 17 +- 3 files changed, 215 insertions(+), 210 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5885a7387f2..46fd981aa77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,9 +115,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher 0.4.4", @@ -126,14 +126,14 @@ dependencies = [ [[package]] name = "aes-gcm" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc3be92e19a7ef47457b8e6f90707e12b6ac5d20c6f3866584fa3be0787d839f" +checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" dependencies = [ "aead 0.4.3", "aes 0.7.5", "cipher 0.3.0", - "ctr 0.7.0", + "ctr 0.8.0", "ghash 0.4.4", "subtle", ] @@ -145,7 +145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead 0.5.2", - "aes 0.8.3", + "aes 0.8.4", "cipher 0.4.4", "ctr 0.9.2", "ghash 0.5.0", @@ -154,9 +154,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" dependencies = [ "cfg-if", "once_cell", @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4b6fb2b432ff223d513db7f908937f63c252bee0af9b82bfd25b0a5dd1eb0d8" +checksum = "ef197eb250c64962003cb08b90b17f0882c192f4a6f2f544809d424fd7cb0e7d" dependencies = [ "alloy-rlp", "bytes", @@ -265,7 +265,7 @@ checksum = "1a047897373be4bbb0224c1afdabca92648dc57a9c9ef6e7b0be3aff7a859c83" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -512,13 +512,13 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" dependencies = [ "concurrent-queue", - "event-listener 4.0.3", - "event-listener-strategy", + "event-listener 5.1.0", + "event-listener-strategy 0.5.0", "futures-core", "pin-project-lite", ] @@ -543,7 +543,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel 2.1.1", + "async-channel 2.2.0", "async-executor", "async-io 2.3.1", "async-lock 3.3.0", @@ -584,8 +584,8 @@ dependencies = [ "futures-io", "futures-lite 2.2.0", "parking", - "polling 3.3.2", - "rustix 0.38.30", + "polling 3.5.0", + "rustix 0.38.31", "slab", "tracing", "windows-sys 0.52.0", @@ -607,7 +607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" dependencies = [ "event-listener 4.0.3", - "event-listener-strategy", + "event-listener-strategy 0.4.0", "pin-project-lite", ] @@ -624,7 +624,7 @@ dependencies = [ "cfg-if", "event-listener 3.1.0", "futures-lite 1.13.0", - "rustix 0.38.30", + "rustix 0.38.31", "windows-sys 0.48.0", ] @@ -640,7 +640,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 0.38.30", + "rustix 0.38.31", "signal-hook-registry", "slab", "windows-sys 0.48.0", @@ -687,7 +687,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -763,7 +763,7 @@ checksum = "823b8bb275161044e2ac7a25879cb3e2480cb403e3943022c7c769c599b756aa" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -1047,7 +1047,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.48", + "syn 2.0.49", "which", ] @@ -1142,7 +1142,7 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel 2.1.1", + "async-channel 2.2.0", "async-lock 3.3.0", "async-task", "fastrand 2.0.1", @@ -1246,9 +1246,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "d32a994c2b3ca201d9b263612a374263f05e7adde37c4707f693dcd375076d1f" [[package]] name = "byte-slice-cast" @@ -1332,9 +1332,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" dependencies = [ "serde", ] @@ -1410,9 +1410,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1562,9 +1562,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5104de16b218eddf8e34ffe2f86f74bfa4e61e95a1b89732fccf6325efd0557" +checksum = "18d59688ad0945eaf6b84cb44fedbe93484c81b48970e98f09db8a22832d7961" dependencies = [ "cfg-if", "cpufeatures", @@ -1627,9 +1627,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1757,9 +1757,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" dependencies = [ "generic-array", "subtle", @@ -1786,15 +1786,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctr" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a232f92a03f37dd7d7dd2adc67166c77e9cd88de5b019b9a9eecfaeaf7bfd481" -dependencies = [ - "cipher 0.3.0", -] - [[package]] name = "ctr" version = "0.8.0" @@ -1848,7 +1839,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2050,7 +2041,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2089,7 +2080,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2109,7 +2100,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2190,7 +2181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac33cb3f99889a57e56a8c6ccb77aaf0cfc7787602b7af09783f736d77314e1" dependencies = [ "aes 0.7.5", - "aes-gcm 0.9.2", + "aes-gcm 0.9.4", "arrayvec", "delay_map", "enr", @@ -2222,7 +2213,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2275,9 +2266,9 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ "curve25519-dalek", "ed25519", @@ -2325,9 +2316,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "elliptic-curve" @@ -2405,7 +2396,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2959,6 +2950,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ad6fd685ce13acd6d9541a30f6db6567a7a24c9ffd4ba2955d29e3f22c8b27" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "event-listener-strategy" version = "0.4.0" @@ -2969,6 +2971,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291" +dependencies = [ + "event-listener 5.1.0", + "pin-project-lite", +] + [[package]] name = "execution_engine_integration" version = "0.1.0" @@ -3057,9 +3069,9 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6267a1fa6f59179ea4afc8e50fd8612a3cc60bc858f786ff877a4a8cb042799" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ "indenter", "once_cell", @@ -3131,9 +3143,9 @@ checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" [[package]] name = "fiat-crypto" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" +checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" [[package]] name = "field-offset" @@ -3357,7 +3369,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3514,7 +3526,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3569,7 +3581,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.2.1", + "indexmap 2.2.3", "slab", "tokio", "tokio-util 0.7.10", @@ -3588,7 +3600,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.2.1", + "indexmap 2.2.3", "slab", "tokio", "tokio-util 0.7.10", @@ -3700,9 +3712,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" [[package]] name = "hex" @@ -3796,7 +3808,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" dependencies = [ - "crypto-mac 0.11.0", + "crypto-mac 0.11.1", "digest 0.9.0", ] @@ -3999,7 +4011,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -4054,12 +4066,11 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", - "futures-channel", "futures-util", "http 1.0.0", "http-body 1.0.0", @@ -4067,14 +4078,13 @@ dependencies = [ "pin-project-lite", "socket2 0.5.5", "tokio", - "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -4262,9 +4272,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.1" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -4315,7 +4325,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.4", + "hermit-abi 0.3.6", "libc", "windows-sys 0.48.0", ] @@ -4386,18 +4396,18 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -4577,9 +4587,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libflate" @@ -4937,7 +4947,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -5525,9 +5535,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -5545,9 +5555,9 @@ dependencies = [ [[package]] name = "mock_instant" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c1a54de846c4006b88b1516731cc1f6026eb5dc4bcb186aa071ef66d40524ec" +checksum = "9366861eb2a2c436c20b12c8dbec5f798cea6b47ad99216be0282942e2c81ea0" [[package]] name = "monitoring_api" @@ -5864,21 +5874,26 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -5887,9 +5902,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -5901,15 +5916,15 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.4", + "hermit-abi 0.3.6", "libc", ] [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] @@ -6005,7 +6020,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -6016,9 +6031,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.2+3.2.1" +version = "300.2.3+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbfad0063610ac26ee79f7484739e2b07555a75c42453b89263830b5c8103bc" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" dependencies = [ "cc", ] @@ -6206,7 +6221,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" dependencies = [ - "crypto-mac 0.11.0", + "crypto-mac 0.11.1", ] [[package]] @@ -6317,7 +6332,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -6365,9 +6380,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "platforms" @@ -6427,14 +6442,14 @@ dependencies = [ [[package]] name = "polling" -version = "3.3.2" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" +checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9" dependencies = [ "cfg-if", "concurrent-queue", "pin-project-lite", - "rustix 0.38.30", + "rustix 0.38.31", "tracing", "windows-sys 0.52.0", ] @@ -6459,7 +6474,7 @@ dependencies = [ "cfg-if", "cpufeatures", "opaque-debug", - "universal-hash 0.4.0", + "universal-hash 0.4.1", ] [[package]] @@ -6539,7 +6554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -6656,7 +6671,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -7016,9 +7031,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64 0.21.7", "bytes", @@ -7044,6 +7059,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -7107,16 +7123,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -7290,9 +7307,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -7308,7 +7325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-webpki", "sct", ] @@ -7328,7 +7345,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -7461,7 +7478,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -7594,7 +7611,7 @@ checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -7626,7 +7643,7 @@ checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -7678,7 +7695,7 @@ version = "0.9.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adf8a49373e98a4c5f0ceb5d05aa7c648d75f63774981ed95b7c7443bbd50c6e" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "itoa", "ryu", "serde", @@ -8043,7 +8060,7 @@ dependencies = [ "chacha20poly1305", "curve25519-dalek", "rand_core", - "ring 0.17.7", + "ring 0.17.8", "rustc_version 0.4.0", "sha2 0.10.8", "subtle", @@ -8235,9 +8252,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "superstruct" @@ -8275,9 +8292,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" dependencies = [ "proc-macro2", "quote", @@ -8390,14 +8407,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall 0.4.1", - "rustix 0.38.30", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -8465,22 +8481,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -8504,13 +8520,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", "libc", + "num-conv", "num_threads", "powerfmt", "serde", @@ -8526,10 +8543,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -8599,9 +8617,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -8633,7 +8651,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -8761,7 +8779,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "serde", "serde_spanned", "toml_datetime", @@ -8774,7 +8792,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "toml_datetime", "winnow", ] @@ -8839,7 +8857,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -9081,9 +9099,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "universal-hash" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" dependencies = [ "generic-array", "subtle", @@ -9400,9 +9418,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -9410,24 +9428,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -9437,9 +9455,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9447,28 +9465,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -9529,9 +9547,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -9565,9 +9583,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "which" @@ -9578,7 +9596,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.30", + "rustix 0.38.31", ] [[package]] @@ -9874,9 +9892,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.35" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -9927,9 +9945,9 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", "rand_core", @@ -10035,7 +10053,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -10055,7 +10073,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -10064,7 +10082,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes 0.8.3", + "aes 0.8.4", "byteorder", "bzip2", "constant_time_eq", diff --git a/beacon_node/lighthouse_network/src/rpc/handler.rs b/beacon_node/lighthouse_network/src/rpc/handler.rs index 03f4761ff08..f4971c18d31 100644 --- a/beacon_node/lighthouse_network/src/rpc/handler.rs +++ b/beacon_node/lighthouse_network/src/rpc/handler.rs @@ -24,7 +24,7 @@ use std::{ task::{Context, Poll}, time::{Duration, Instant}, }; -use tokio::time::{sleep_until, Instant as TInstant, Sleep}; +use tokio::time::{sleep, Sleep}; use tokio_util::time::{delay_queue, DelayQueue}; use types::{EthSpec, ForkContext}; @@ -32,7 +32,7 @@ use types::{EthSpec, ForkContext}; const IO_ERROR_RETRIES: u8 = 3; /// Maximum time given to the handler to perform shutdown operations. -const SHUTDOWN_TIMEOUT_SECS: u8 = 15; +const SHUTDOWN_TIMEOUT_SECS: u64 = 15; /// Maximum number of simultaneous inbound substreams we keep for this peer. const MAX_INBOUND_SUBSTREAMS: usize = 32; @@ -266,9 +266,9 @@ where self.dial_queue.push((id, OutboundRequest::Goodbye(reason))); } - self.state = HandlerState::ShuttingDown(Box::pin(sleep_until( - TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64), - ))); + self.state = HandlerState::ShuttingDown(Box::pin(sleep(Duration::from_secs( + SHUTDOWN_TIMEOUT_SECS, + )))); } } @@ -349,23 +349,7 @@ where } fn connection_keep_alive(&self) -> bool { - // Check that we don't have outbound items pending for dialing, nor dialing, nor - // established. Also check that there are no established inbound substreams. - // Errors and events need to be reported back, so check those too. - match self.state { - HandlerState::ShuttingDown(_) => { - !self.dial_queue.is_empty() - || !self.outbound_substreams.is_empty() - || !self.inbound_substreams.is_empty() - || !self.events_out.is_empty() - || !self.dial_negotiated != 0 - } - HandlerState::Deactivated => { - // Regardless of events, the timeout has expired. Force the disconnect. - false - } - _ => true, - } + !matches!(self.state, HandlerState::Deactivated) } fn poll( @@ -395,7 +379,7 @@ where match delay.as_mut().poll(cx) { Poll::Ready(_) => { self.state = HandlerState::Deactivated; - debug!(self.log, "Handler deactivated"); + debug!(self.log, "Shutdown timeout elapsed, Handler deactivated"); return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( HandlerEvent::Close(RPCError::Disconnected), )); @@ -844,6 +828,8 @@ where && self.events_out.is_empty() && self.dial_negotiated == 0 { + debug!(self.log, "Goodbye sent, Handler deactivated"); + self.state = HandlerState::Deactivated; return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( HandlerEvent::Close(RPCError::Disconnected), )); diff --git a/beacon_node/lighthouse_network/src/rpc/mod.rs b/beacon_node/lighthouse_network/src/rpc/mod.rs index 3606438fb99..e22e5273866 100644 --- a/beacon_node/lighthouse_network/src/rpc/mod.rs +++ b/beacon_node/lighthouse_network/src/rpc/mod.rs @@ -223,7 +223,7 @@ where fn handle_established_inbound_connection( &mut self, - _connection_id: ConnectionId, + connection_id: ConnectionId, peer_id: PeerId, _local_addr: &libp2p::Multiaddr, _remote_addr: &libp2p::Multiaddr, @@ -238,9 +238,9 @@ where }, (), ); - // NOTE: this is needed because PeerIds have interior mutability. - let peer_repr = peer_id.to_string(); - let log = self.log.new(slog::o!("peer_id" => peer_repr)); + let log = self + .log + .new(slog::o!("peer_id" => peer_id.to_string(), "connection_id" => connection_id.to_string())); let handler = RPCHandler::new( protocol, self.fork_context.clone(), @@ -253,7 +253,7 @@ where fn handle_established_outbound_connection( &mut self, - _connection_id: ConnectionId, + connection_id: ConnectionId, peer_id: PeerId, _addr: &libp2p::Multiaddr, _role_override: libp2p::core::Endpoint, @@ -269,9 +269,10 @@ where (), ); - // NOTE: this is needed because PeerIds have interior mutability. - let peer_repr = peer_id.to_string(); - let log = self.log.new(slog::o!("peer_id" => peer_repr)); + let log = self + .log + .new(slog::o!("peer_id" => peer_id.to_string(), "connection_id" => connection_id.to_string())); + let handler = RPCHandler::new( protocol, self.fork_context.clone(), From 50c423ad88aa584edf9aca18e5d1d1ee26b9ad1f Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 20 Feb 2024 15:19:17 +1100 Subject: [PATCH 13/21] Revert libp2p metrics (#4870) (#5265) * Revert "improve libp2p connected peer metrics (#4870)" This reverts commit 0c3fef59b3c6697ee84e4f89a8537c097eacfc5a. --- beacon_node/http_api/src/lib.rs | 10 ++- .../lighthouse_network/src/discovery/mod.rs | 8 +-- beacon_node/lighthouse_network/src/metrics.rs | 68 ++++++++++++++++--- .../src/peer_manager/mod.rs | 18 +++++ .../src/peer_manager/network_behaviour.rs | 26 +++---- common/lighthouse_metrics/src/lib.rs | 10 --- common/system_health/src/lib.rs | 25 ++----- 7 files changed, 100 insertions(+), 65 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index b39450d7354..a9b245e7987 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -68,7 +68,7 @@ use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use sysinfo::{System, SystemExt}; -use system_health::{observe_nat, observe_system_health_bn}; +use system_health::observe_system_health_bn; use task_spawner::{Priority, TaskSpawner}; use tokio::sync::{ mpsc::{Sender, UnboundedSender}, @@ -3965,7 +3965,13 @@ pub fn serve( .and(warp::path::end()) .then(|task_spawner: TaskSpawner| { task_spawner.blocking_json_task(Priority::P1, move || { - Ok(api_types::GenericResponse::from(observe_nat())) + Ok(api_types::GenericResponse::from( + lighthouse_network::metrics::NAT_OPEN + .as_ref() + .map(|v| v.get()) + .unwrap_or(0) + != 0, + )) }) }); diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index c1781c85832..6659ba1d26f 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -1003,13 +1003,11 @@ impl NetworkBehaviour for Discovery { } discv5::Event::SocketUpdated(socket_addr) => { info!(self.log, "Address updated"; "ip" => %socket_addr.ip(), "udp_port" => %socket_addr.port()); - // We have SOCKET_UPDATED messages. This occurs when discovery has a majority of - // users reporting an external port and our ENR gets updated. - // Which means we are able to do NAT traversal. - metrics::set_gauge_vec(&metrics::NAT_OPEN, &["discv5"], 1); - + metrics::inc_counter(&metrics::ADDRESS_UPDATE_COUNT); + metrics::check_nat(); // Discv5 will have updated our local ENR. We save the updated version // to disk. + if (self.update_ports.tcp4 && socket_addr.is_ipv4()) || (self.update_ports.tcp6 && socket_addr.is_ipv6()) { diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index ada48d6ebd0..ae02b689d81 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -1,17 +1,29 @@ pub use lighthouse_metrics::*; lazy_static! { - pub static ref NAT_OPEN: Result = try_create_int_gauge_vec( + pub static ref NAT_OPEN: Result = try_create_int_counter( "nat_open", - "An estimate indicating if the local node is reachable from external nodes", - &["protocol"] + "An estimate indicating if the local node is exposed to the internet." ); pub static ref ADDRESS_UPDATE_COUNT: Result = try_create_int_counter( "libp2p_address_update_total", "Count of libp2p socked updated events (when our view of our IP address has changed)" ); - pub static ref PEERS_CONNECTED: Result = - try_create_int_gauge_vec("libp2p_peers", "Count of libp2p peers currently connected", &["direction", "transport"]); + pub static ref PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_peers", + "Count of libp2p peers currently connected" + ); + + pub static ref TCP_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_tcp_peers", + "Count of libp2p peers currently connected via TCP" + ); + + pub static ref QUIC_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_quic_peers", + "Count of libp2p peers currently connected via QUIC" + ); + pub static ref PEER_CONNECT_EVENT_COUNT: Result = try_create_int_counter( "libp2p_peer_connect_event_total", "Count of libp2p peer connect events (not the current number of connected peers)" @@ -20,10 +32,13 @@ lazy_static! { "libp2p_peer_disconnect_event_total", "Count of libp2p peer disconnect events" ); - pub static ref DISCOVERY_BYTES: Result = try_create_int_gauge_vec( - "discovery_bytes", - "The number of bytes sent and received in discovery", - &["direction"] + pub static ref DISCOVERY_SENT_BYTES: Result = try_create_int_gauge( + "discovery_sent_bytes", + "The number of bytes sent in discovery" + ); + pub static ref DISCOVERY_RECV_BYTES: Result = try_create_int_gauge( + "discovery_recv_bytes", + "The number of bytes received in discovery" ); pub static ref DISCOVERY_QUEUE: Result = try_create_int_gauge( "discovery_queue_size", @@ -120,6 +135,17 @@ lazy_static! { &["type"] ); + /* + * Inbound/Outbound peers + */ + /// The number of peers that dialed us. + pub static ref NETWORK_INBOUND_PEERS: Result = + try_create_int_gauge("network_inbound_peers","The number of peers that are currently connected that have dialed us."); + + /// The number of peers that we dialed us. + pub static ref NETWORK_OUTBOUND_PEERS: Result = + try_create_int_gauge("network_outbound_peers","The number of peers that are currently connected that we dialed."); + /* * Peer Reporting */ @@ -130,11 +156,31 @@ lazy_static! { ); } +/// Checks if we consider the NAT open. +/// +/// Conditions for an open NAT: +/// 1. We have 1 or more SOCKET_UPDATED messages. This occurs when discovery has a majority of +/// users reporting an external port and our ENR gets updated. +/// 2. We have 0 SOCKET_UPDATED messages (can be true if the port was correct on boot), then we +/// rely on whether we have any inbound messages. If we have no socket update messages, but +/// manage to get at least one inbound peer, we are exposed correctly. +pub fn check_nat() { + // NAT is already deemed open. + if NAT_OPEN.as_ref().map(|v| v.get()).unwrap_or(0) != 0 { + return; + } + if ADDRESS_UPDATE_COUNT.as_ref().map(|v| v.get()).unwrap_or(0) != 0 + || NETWORK_INBOUND_PEERS.as_ref().map(|v| v.get()).unwrap_or(0) != 0_i64 + { + inc_counter(&NAT_OPEN); + } +} + pub fn scrape_discovery_metrics() { let metrics = discv5::metrics::Metrics::from(discv5::Discv5::::raw_metrics()); set_float_gauge(&DISCOVERY_REQS, metrics.unsolicited_requests_per_second); set_gauge(&DISCOVERY_SESSIONS, metrics.active_sessions as i64); - set_gauge_vec(&DISCOVERY_BYTES, &["inbound"], metrics.bytes_recv as i64); - set_gauge_vec(&DISCOVERY_BYTES, &["outbound"], metrics.bytes_sent as i64); + set_gauge(&DISCOVERY_SENT_BYTES, metrics.bytes_sent as i64); + set_gauge(&DISCOVERY_RECV_BYTES, metrics.bytes_recv as i64); } diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 8227c3420b2..e4976a0d374 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -726,14 +726,29 @@ impl PeerManager { return; } + let mut connected_peer_count = 0; + let mut inbound_connected_peers = 0; + let mut outbound_connected_peers = 0; let mut clients_per_peer = HashMap::new(); for (_peer, peer_info) in self.network_globals.peers.read().connected_peers() { + connected_peer_count += 1; + if let PeerConnectionStatus::Connected { n_in, .. } = peer_info.connection_status() { + if *n_in > 0 { + inbound_connected_peers += 1; + } else { + outbound_connected_peers += 1; + } + } *clients_per_peer .entry(peer_info.client().kind.to_string()) .or_default() += 1; } + metrics::set_gauge(&metrics::PEERS_CONNECTED, connected_peer_count); + metrics::set_gauge(&metrics::NETWORK_INBOUND_PEERS, inbound_connected_peers); + metrics::set_gauge(&metrics::NETWORK_OUTBOUND_PEERS, outbound_connected_peers); + for client_kind in ClientKind::iter() { let value = clients_per_peer.get(&client_kind.to_string()).unwrap_or(&0); metrics::set_gauge_vec( @@ -838,8 +853,11 @@ impl PeerManager { // start a ping and status timer for the peer self.status_peers.insert(*peer_id); + let connected_peers = self.network_globals.connected_peers() as i64; + // increment prometheus metrics metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + metrics::set_gauge(&metrics::PEERS_CONNECTED, connected_peers); true } diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 1c9274af233..cb60906f632 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -154,8 +154,6 @@ impl NetworkBehaviour for PeerManager { self.on_dial_failure(peer_id); } FromSwarm::ExternalAddrConfirmed(_) => { - // We have an external address confirmed, means we are able to do NAT traversal. - metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p"], 1); // TODO: we likely want to check this against our assumed external tcp // address } @@ -245,14 +243,14 @@ impl PeerManager { self.events.push(PeerManagerEvent::MetaData(peer_id)); } + // Check NAT if metrics are enabled + if self.network_globals.local_enr.read().udp4().is_some() { + metrics::check_nat(); + } + // increment prometheus metrics if self.metrics_enabled { let remote_addr = endpoint.get_remote_address(); - let direction = if endpoint.is_dialer() { - "outbound" - } else { - "inbound" - }; match remote_addr.iter().find(|proto| { matches!( proto, @@ -260,10 +258,10 @@ impl PeerManager { ) }) { Some(multiaddr::Protocol::QuicV1) => { - metrics::inc_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "quic"]); + metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); } Some(multiaddr::Protocol::Tcp(_)) => { - metrics::inc_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "tcp"]); + metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); } Some(_) => unreachable!(), None => { @@ -341,12 +339,6 @@ impl PeerManager { let remote_addr = endpoint.get_remote_address(); // Update the prometheus metrics if self.metrics_enabled { - let direction = if endpoint.is_dialer() { - "outbound" - } else { - "inbound" - }; - match remote_addr.iter().find(|proto| { matches!( proto, @@ -354,10 +346,10 @@ impl PeerManager { ) }) { Some(multiaddr::Protocol::QuicV1) => { - metrics::dec_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "quic"]); + metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); } Some(multiaddr::Protocol::Tcp(_)) => { - metrics::dec_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "tcp"]); + metrics::dec_gauge(&metrics::TCP_PEERS_CONNECTED); } // If it's an unknown protocol we already logged when connection was established. _ => {} diff --git a/common/lighthouse_metrics/src/lib.rs b/common/lighthouse_metrics/src/lib.rs index ba9900cc507..5d25bb313f6 100644 --- a/common/lighthouse_metrics/src/lib.rs +++ b/common/lighthouse_metrics/src/lib.rs @@ -244,16 +244,6 @@ pub fn inc_counter_vec(int_counter_vec: &Result, name: &[&str]) { } } -/// Sets the `int_counter_vec` with the given `name` to the `amount`, -/// should only be called with `ammount`s equal or above the current value -/// as per Prometheus spec, the `counter` type should only go up. -pub fn set_counter_vec_by(int_counter_vec: &Result, name: &[&str], amount: u64) { - if let Some(counter) = get_int_counter(int_counter_vec, name) { - counter.reset(); - counter.inc_by(amount); - } -} - pub fn inc_counter_vec_by(int_counter_vec: &Result, name: &[&str], amount: u64) { if let Some(counter) = get_int_counter(int_counter_vec, name) { counter.inc_by(amount); diff --git a/common/system_health/src/lib.rs b/common/system_health/src/lib.rs index ec64ce31ad3..d10540e506c 100644 --- a/common/system_health/src/lib.rs +++ b/common/system_health/src/lib.rs @@ -198,25 +198,6 @@ pub fn observe_system_health_vc( } } -/// Observes if NAT traversal is possible. -pub fn observe_nat() -> bool { - let discv5_nat = lighthouse_network::metrics::get_int_gauge( - &lighthouse_network::metrics::NAT_OPEN, - &["discv5"], - ) - .map(|g| g.get() == 1) - .unwrap_or_default(); - - let libp2p_nat = lighthouse_network::metrics::get_int_gauge( - &lighthouse_network::metrics::NAT_OPEN, - &["libp2p"], - ) - .map(|g| g.get() == 1) - .unwrap_or_default(); - - discv5_nat && libp2p_nat -} - /// Observes the Beacon Node system health. pub fn observe_system_health_bn( sysinfo: Arc>, @@ -242,7 +223,11 @@ pub fn observe_system_health_bn( .unwrap_or_else(|| (String::from("None"), 0, 0)); // Determine if the NAT is open or not. - let nat_open = observe_nat(); + let nat_open = lighthouse_network::metrics::NAT_OPEN + .as_ref() + .map(|v| v.get()) + .unwrap_or(0) + != 0; SystemHealthBN { system_health, From b5bae6e7a2f52310617a3b93ab7579f3dc9d7880 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 21 Feb 2024 09:12:52 +1100 Subject: [PATCH 14/21] Release v5.0.0 (#5254) * Bump versions --- Cargo.lock | 8 ++++---- beacon_node/Cargo.toml | 2 +- boot_node/Cargo.toml | 2 +- common/lighthouse_version/src/lib.rs | 4 ++-- lcli/Cargo.toml | 2 +- lighthouse/Cargo.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46fd981aa77..aa17e2f4ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "beacon_node" -version = "4.6.0" +version = "5.0.0" dependencies = [ "beacon_chain", "clap", @@ -1194,7 +1194,7 @@ dependencies = [ [[package]] name = "boot_node" -version = "4.6.0" +version = "5.0.0" dependencies = [ "beacon_node", "clap", @@ -4524,7 +4524,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lcli" -version = "4.6.0" +version = "5.0.0" dependencies = [ "account_utils", "beacon_chain", @@ -5100,7 +5100,7 @@ dependencies = [ [[package]] name = "lighthouse" -version = "4.6.0" +version = "5.0.0" dependencies = [ "account_manager", "account_utils", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 8428a30a3b1..f960251e7a3 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "beacon_node" -version = "4.6.0" +version = "5.0.0" authors = [ "Paul Hauner ", "Age Manning "] edition = { workspace = true } diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index af11723487f..10759f94306 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!( // NOTE: using --match instead of --exclude for compatibility with old Git "--match=thiswillnevermatchlol" ], - prefix = "Lighthouse/v4.6.0-", - fallback = "Lighthouse/v4.6.0" + prefix = "Lighthouse/v5.0.0-", + fallback = "Lighthouse/v5.0.0" ); /// Returns `VERSION`, but with platform information appended to the end. diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 1bba8242c54..3acf3909b3b 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lcli" description = "Lighthouse CLI (modeled after zcli)" -version = "4.6.0" +version = "5.0.0" authors = ["Paul Hauner "] edition = { workspace = true } diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index b14227e7edf..ffa4727d7f2 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lighthouse" -version = "4.6.0" +version = "5.0.0" authors = ["Sigma Prime "] edition = { workspace = true } autotests = false From 13956a07416ba00d514c0895fdc1dc2f1f74de7b Mon Sep 17 00:00:00 2001 From: chonghe <44791194+chong-he@users.noreply.github.com> Date: Fri, 23 Feb 2024 23:31:55 +0800 Subject: [PATCH 15/21] Add build instructions for Fedora/RHEL/CentOS (#5225) * Add dependencies --- book/src/installation-source.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/book/src/installation-source.md b/book/src/installation-source.md index 58e6917eca9..75ed6121ab9 100644 --- a/book/src/installation-source.md +++ b/book/src/installation-source.md @@ -42,6 +42,16 @@ sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clan After this, you are ready to [build Lighthouse](#build-lighthouse). +#### Fedora/RHEL/CentOS + +Install the following packages: + +```bash +yum -y install git make perl clang cmake +``` + +After this, you are ready to [build Lighthouse](#build-lighthouse). + #### macOS 1. Install the [Homebrew][] package manager. From f08e8f56330ca56106e10b62b389a6019857cc96 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 26 Feb 2024 14:10:59 +1100 Subject: [PATCH 16/21] Run `apt update` before install. (#5295) * Run `apt update` before install. --- .github/workflows/test-suite.yml | 2 +- book/src/installation-source.md | 2 +- book/src/pi.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 28b8ec29e33..50ccb02ad3a 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -408,7 +408,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install dependencies - run: sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang + run: sudo apt update && sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang - name: Use Rust beta run: rustup override set beta - name: Run make diff --git a/book/src/installation-source.md b/book/src/installation-source.md index 75ed6121ab9..c2f5861576d 100644 --- a/book/src/installation-source.md +++ b/book/src/installation-source.md @@ -28,7 +28,7 @@ operating system. Install the following packages: ```bash -sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang +sudo apt update && sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang ``` > Tips: diff --git a/book/src/pi.md b/book/src/pi.md index 550415240b4..7ccfe6a02a0 100644 --- a/book/src/pi.md +++ b/book/src/pi.md @@ -22,7 +22,7 @@ terminal and an Internet connection are necessary. Install the Ubuntu dependencies: ```bash -sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang +sudo apt update && sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang ``` > Tips: From de6ede163c46fca2fe4855ee58c28d6671a32eca Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 26 Feb 2024 16:19:35 +1100 Subject: [PATCH 17/21] Delete ancient, unused HTTP docs (#5281) * Delete ancient, unused HTTP docs --- book/src/http.md | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 book/src/http.md diff --git a/book/src/http.md b/book/src/http.md deleted file mode 100644 index 82a688586b0..00000000000 --- a/book/src/http.md +++ /dev/null @@ -1,33 +0,0 @@ -# HTTP API - -[OpenAPI Specification](https://ethereum.github.io/beacon-APIs/) - -## Beacon Node - -A Lighthouse beacon node can be configured to expose a HTTP server by supplying the `--http` flag. The default listen address is `localhost:5052`. - -The following CLI flags control the HTTP server: - -- `--http`: enable the HTTP server (required even if the following flags are - provided). -- `--http-port`: specify the listen port of the server. -- `--http-address`: specify the listen address of the server. - -The schema of the API aligns with the standard Ethereum Beacon Node API as defined -at [github.com/ethereum/beacon-APIs](https://github.com/ethereum/beacon-APIs). -It is an easy-to-use RESTful HTTP/JSON API. An interactive specification is -available [here](https://ethereum.github.io/beacon-APIs/). - -## Troubleshooting - -### HTTP API is unavailable or refusing connections - -Ensure the `--http` flag has been supplied at the CLI. - -You can quickly check that the HTTP endpoint is up using `curl`: - -``` -curl "localhost:5052/beacon/head" - -{"slot":37934,"block_root":"0x4d3ae7ebe8c6ef042db05958ec76e8f7be9d412a67a0defa6420a677249afdc7","state_root":"0x1c86b13ffc70a41e410eccce20d33f1fe59d148585ea27c2afb4060f75fe6be2","finalized_slot":37856,"finalized_block_root":"0xbdae152b62acef1e5c332697567d2b89e358628790b8273729096da670b23e86","justified_slot":37888,"justified_block_root":"0x01c2f516a407d8fdda23cad4ed4381e4ab8913d638f935a2fe9bd00d6ced5ec4","previous_justified_slot":37856,"previous_justified_block_root":"0xbdae152b62acef1e5c332697567d2b89e358628790b8273729096da670b23e86"} -``` From 3ab9d3a84efc56539cb59af1b7cbb112b7849bc5 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 26 Feb 2024 10:49:39 +0530 Subject: [PATCH 18/21] Add a cli option for the snapshot cache size (#5270) * Add a cli option for snapshot cache size * Remove junk * Make snapshot_cache module public * lint * Update docs --- beacon_node/beacon_chain/src/builder.rs | 5 +++-- beacon_node/beacon_chain/src/chain_config.rs | 3 +++ beacon_node/beacon_chain/src/lib.rs | 2 +- .../beacon_chain/src/snapshot_cache.rs | 3 ++- beacon_node/src/cli.rs | 7 +++++++ beacon_node/src/config.rs | 3 +++ book/src/help_bn.md | 3 +++ lighthouse/tests/beacon_node.rs | 20 +++++++++++++++++++ 8 files changed, 42 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index c75c3f695b3..dd4b612f60b 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -12,7 +12,7 @@ use crate::light_client_server_cache::LightClientServerCache; use crate::migrate::{BackgroundMigrator, MigratorConfig}; use crate::persisted_beacon_chain::PersistedBeaconChain; use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache}; -use crate::snapshot_cache::{SnapshotCache, DEFAULT_SNAPSHOT_CACHE_SIZE}; +use crate::snapshot_cache::SnapshotCache; use crate::timeout_rw_lock::TimeoutRwLock; use crate::validator_monitor::{ValidatorMonitor, ValidatorMonitorConfig}; use crate::validator_pubkey_cache::ValidatorPubkeyCache; @@ -870,6 +870,7 @@ where let head_for_snapshot_cache = head_snapshot.clone(); let canonical_head = CanonicalHead::new(fork_choice, Arc::new(head_snapshot)); let shuffling_cache_size = self.chain_config.shuffling_cache_size; + let snapshot_cache_size = self.chain_config.snapshot_cache_size; // Calculate the weak subjectivity point in which to backfill blocks to. let genesis_backfill_slot = if self.chain_config.genesis_backfill { @@ -946,7 +947,7 @@ where event_handler: self.event_handler, head_tracker, snapshot_cache: TimeoutRwLock::new(SnapshotCache::new( - DEFAULT_SNAPSHOT_CACHE_SIZE, + snapshot_cache_size, head_for_snapshot_cache, )), shuffling_cache: TimeoutRwLock::new(ShufflingCache::new( diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 23e17a6efad..36481b4dcd0 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -72,6 +72,8 @@ pub struct ChainConfig { pub optimistic_finalized_sync: bool, /// The size of the shuffling cache, pub shuffling_cache_size: usize, + /// The size of the snapshot cache. + pub snapshot_cache_size: usize, /// If using a weak-subjectivity sync, whether we should download blocks all the way back to /// genesis. pub genesis_backfill: bool, @@ -112,6 +114,7 @@ impl Default for ChainConfig { // This value isn't actually read except in tests. optimistic_finalized_sync: true, shuffling_cache_size: crate::shuffling_cache::DEFAULT_CACHE_SIZE, + snapshot_cache_size: crate::snapshot_cache::DEFAULT_SNAPSHOT_CACHE_SIZE, genesis_backfill: false, always_prepare_payload: false, progressive_balances_mode: ProgressiveBalancesMode::Fast, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 522009b1b27..529f269be10 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -50,7 +50,7 @@ mod pre_finalization_cache; pub mod proposer_prep_service; pub mod schema_change; pub mod shuffling_cache; -mod snapshot_cache; +pub mod snapshot_cache; pub mod state_advance_timer; pub mod sync_committee_rewards; pub mod sync_committee_verification; diff --git a/beacon_node/beacon_chain/src/snapshot_cache.rs b/beacon_node/beacon_chain/src/snapshot_cache.rs index d2846c08569..765ed0cb2aa 100644 --- a/beacon_node/beacon_chain/src/snapshot_cache.rs +++ b/beacon_node/beacon_chain/src/snapshot_cache.rs @@ -9,7 +9,7 @@ use types::{ }; /// The default size of the cache. -pub const DEFAULT_SNAPSHOT_CACHE_SIZE: usize = 4; +pub const DEFAULT_SNAPSHOT_CACHE_SIZE: usize = 3; /// The minimum block delay to clone the state in the cache instead of removing it. /// This helps keep block processing fast during re-orgs from late blocks. @@ -174,6 +174,7 @@ impl SnapshotCache { self.snapshots.iter().map(|s| s.beacon_block_root).collect() } + #[allow(clippy::len_without_is_empty)] /// The number of snapshots contained in `self`. pub fn len(&self) -> usize { self.snapshots.len() diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index fa4edc34d22..1a8e0194f6e 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -622,6 +622,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("Specifies how many states from the freezer database should cache in memory [default: 1]") .takes_value(true) ) + .arg( + Arg::with_name("state-cache-size") + .long("state-cache-size") + .value_name("STATE_CACHE_SIZE") + .help("Specifies the size of the snapshot cache [default: 3]") + .takes_value(true) + ) /* * Execution Layer Integration */ diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index fc6132f8c9b..ba8430aceae 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -170,6 +170,9 @@ pub fn get_config( if let Some(cache_size) = clap_utils::parse_optional(cli_args, "shuffling-cache-size")? { client_config.chain.shuffling_cache_size = cache_size; } + if let Some(cache_size) = clap_utils::parse_optional(cli_args, "state-cache-size")? { + client_config.chain.snapshot_cache_size = cache_size; + } /* * Prometheus metrics HTTP server diff --git a/book/src/help_bn.md b/book/src/help_bn.md index b2a922020f5..e55c34a9ff9 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -461,6 +461,9 @@ OPTIONS: --slots-per-restore-point Specifies how often a freezer DB restore point should be stored. Cannot be changed after initialization. [default: 8192 (mainnet) or 64 (minimal)] + --state-cache-size + Specifies the size of the snapshot cache [default: 3] + --suggested-fee-recipient Emergency fallback fee recipient for use in case the validator client does not have one configured. You should set this flag on the validator client instead of (or in addition to) setting it here. diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 94996eb1a26..b61ce5922b3 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -172,6 +172,26 @@ fn shuffling_cache_set() { .with_config(|config| assert_eq!(config.chain.shuffling_cache_size, 500)); } +#[test] +fn snapshot_cache_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.snapshot_cache_size, + beacon_node::beacon_chain::snapshot_cache::DEFAULT_SNAPSHOT_CACHE_SIZE + ) + }); +} + +#[test] +fn snapshot_cache_set() { + CommandLineTest::new() + .flag("state-cache-size", Some("500")) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.chain.snapshot_cache_size, 500)); +} + #[test] fn fork_choice_before_proposal_timeout_default() { CommandLineTest::new() From 744c598b1cd271b63345f42337d2c408f5a8c4ff Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 26 Feb 2024 18:45:17 +1100 Subject: [PATCH 19/21] Fix typos and make block hash calculation public (#5275) * Fix typo in `verify_payload_block_hash` * Make calculate_execution_block_hash public again --- .../execution_layer/src/engine_api/new_payload_request.rs | 4 ++-- beacon_node/execution_layer/src/lib.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs index b1385399e89..b9527ed09db 100644 --- a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs +++ b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs @@ -86,7 +86,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { /// /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#modified-verify_and_notify_new_payload pub fn perform_optimistic_sync_verifications(&self) -> Result<(), Error> { - self.verfiy_payload_block_hash()?; + self.verify_payload_block_hash()?; self.verify_versioned_hashes()?; Ok(()) @@ -98,7 +98,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { /// /// Equivalent to `is_valid_block_hash` in the spec: /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#is_valid_block_hash - pub fn verfiy_payload_block_hash(&self) -> Result<(), Error> { + pub fn verify_payload_block_hash(&self) -> Result<(), Error> { let payload = self.execution_payload_ref(); let parent_beacon_block_root = self.parent_beacon_block_root().ok().cloned(); diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 664ceabb6cd..69b84adbb8f 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -7,6 +7,7 @@ use crate::payload_cache::PayloadCache; use arc_swap::ArcSwapOption; use auth::{strip_prefix, Auth, JwtKey}; +pub use block_hash::calculate_execution_block_hash; use builder_client::BuilderHttpClient; pub use engine_api::EngineCapabilities; use engine_api::Error as ApiError; From d36241b4a12719800d04673d8f700ccf0d9253a8 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 27 Feb 2024 17:32:48 +1100 Subject: [PATCH 20/21] Track multiaddr in connection status (#5308) * Record the multiaddr for connected peers --- .../src/peer_manager/peerdb.rs | 4 +- .../src/peer_manager/peerdb/peer_info.rs | 39 ++++++++++++------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index a6bf3ffecce..8aef0bad420 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -756,8 +756,8 @@ impl PeerDB { // Update the connection state match direction { - ConnectionDirection::Incoming => info.connect_ingoing(Some(seen_address)), - ConnectionDirection::Outgoing => info.connect_outgoing(Some(seen_address)), + ConnectionDirection::Incoming => info.connect_ingoing(seen_address), + ConnectionDirection::Outgoing => info.connect_outgoing(seen_address), } } diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs index 44c54511ddc..c5e13c51506 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs @@ -307,13 +307,13 @@ impl PeerInfo { /// Checks if the peer is outbound-only pub fn is_outbound_only(&self) -> bool { - matches!(self.connection_status, Connected {n_in, n_out} if n_in == 0 && n_out > 0) + matches!(self.connection_status, Connected {n_in, n_out, ..} if n_in == 0 && n_out > 0) } /// Returns the number of connections with this peer. pub fn connections(&self) -> (u8, u8) { match self.connection_status { - Connected { n_in, n_out } => (n_in, n_out), + Connected { n_in, n_out, .. } => (n_in, n_out), _ => (0, 0), } } @@ -421,7 +421,9 @@ impl PeerInfo { /// Modifies the status to Connected and increases the number of ingoing /// connections by one - pub(super) fn connect_ingoing(&mut self, seen_multiaddr: Option) { + pub(super) fn connect_ingoing(&mut self, multiaddr: Multiaddr) { + self.seen_multiaddrs.insert(multiaddr.clone()); + match &mut self.connection_status { Connected { n_in, .. } => *n_in += 1, Disconnected { .. } @@ -429,19 +431,20 @@ impl PeerInfo { | Dialing { .. } | Disconnecting { .. } | Unknown => { - self.connection_status = Connected { n_in: 1, n_out: 0 }; + self.connection_status = Connected { + n_in: 1, + n_out: 0, + multiaddr, + }; self.connection_direction = Some(ConnectionDirection::Incoming); } } - - if let Some(multiaddr) = seen_multiaddr { - self.seen_multiaddrs.insert(multiaddr); - } } /// Modifies the status to Connected and increases the number of outgoing /// connections by one - pub(super) fn connect_outgoing(&mut self, seen_multiaddr: Option) { + pub(super) fn connect_outgoing(&mut self, multiaddr: Multiaddr) { + self.seen_multiaddrs.insert(multiaddr.clone()); match &mut self.connection_status { Connected { n_out, .. } => *n_out += 1, Disconnected { .. } @@ -449,13 +452,14 @@ impl PeerInfo { | Dialing { .. } | Disconnecting { .. } | Unknown => { - self.connection_status = Connected { n_in: 0, n_out: 1 }; + self.connection_status = Connected { + n_in: 0, + n_out: 1, + multiaddr, + }; self.connection_direction = Some(ConnectionDirection::Outgoing); } } - if let Some(multiaddr) = seen_multiaddr { - self.seen_multiaddrs.insert(multiaddr); - } } #[cfg(test)] @@ -487,6 +491,8 @@ pub enum ConnectionDirection { pub enum PeerConnectionStatus { /// The peer is connected. Connected { + /// The multiaddr that we are connected via. + multiaddr: Multiaddr, /// number of ingoing connections. n_in: u8, /// number of outgoing connections. @@ -522,7 +528,12 @@ impl Serialize for PeerConnectionStatus { fn serialize(&self, serializer: S) -> Result { let mut s = serializer.serialize_struct("connection_status", 6)?; match self { - Connected { n_in, n_out } => { + Connected { + n_in, + n_out, + multiaddr, + } => { + s.serialize_field("multiaddr", multiaddr)?; s.serialize_field("status", "connected")?; s.serialize_field("connections_in", n_in)?; s.serialize_field("connections_out", n_out)?; From abd99652b4571af05ecc19f107dd5f740476efe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Tue, 27 Feb 2024 07:29:18 +0000 Subject: [PATCH 21/21] remove nat module and use libp2p upnp (#4840) * remove nat module and use libp2p upnp * update Cargo.lock * remove no longer used dependencies * restore nat module refactored * log successful mapping * only activate upnp if config enabled reduce logs to debug! * Merge branch 'unstable' of https://github.com/sigp/lighthouse into libp2p-nat * Merge branch 'unstable' of https://github.com/sigp/lighthouse into libp2p-nat * Merge branch 'unstable' of https://github.com/sigp/lighthouse into libp2p-nat * address review * Merge branch 'unstable' of https://github.com/sigp/lighthouse into libp2p-nat * Merge branch 'unstable' of https://github.com/sigp/lighthouse into libp2p-nat * Merge branch 'unstable' of https://github.com/sigp/lighthouse into libp2p-nat * address review --- Cargo.lock | 73 ++--- Cargo.toml | 1 + beacon_node/lighthouse_network/Cargo.toml | 2 +- .../src/service/behaviour.rs | 3 + .../lighthouse_network/src/service/mod.rs | 51 +++- .../lighthouse_network/src/service/utils.rs | 5 +- beacon_node/network/Cargo.toml | 6 +- beacon_node/network/src/nat.rs | 269 ++++-------------- beacon_node/network/src/service.rs | 64 ++--- 9 files changed, 151 insertions(+), 323 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa17e2f4ceb..643b275d91a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,9 +299,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "arbitrary" @@ -1347,7 +1347,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver 1.0.21", + "semver 1.0.22", "serde", "serde_json", "thiserror", @@ -4129,17 +4129,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "if-addrs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2273e421f7c4f0fc99e1934fe4776f59d8df2972f4199d703fc0da9f2a9f73de" -dependencies = [ - "if-addrs-sys", - "libc", - "winapi", -] - [[package]] name = "if-addrs" version = "0.10.2" @@ -4150,16 +4139,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "if-addrs-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de74b9dd780476e837e5eb5ab7c88b49ed304126e412030a0adba99c8efe79ea" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "if-watch" version = "3.2.0" @@ -4170,7 +4149,7 @@ dependencies = [ "core-foundation", "fnv", "futures", - "if-addrs 0.10.2", + "if-addrs", "ipnet", "log", "rtnetlink", @@ -4348,6 +4327,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.6", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -5726,6 +5716,7 @@ dependencies = [ name = "network" version = "0.2.0" dependencies = [ + "anyhow", "beacon_chain", "beacon_processor", "delay_map", @@ -5741,7 +5732,6 @@ dependencies = [ "futures", "genesis", "hex", - "if-addrs 0.6.7", "igd-next", "itertools", "lazy_static", @@ -5920,15 +5910,6 @@ dependencies = [ "libc", ] -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - [[package]] name = "object" version = "0.32.2" @@ -7265,7 +7246,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.21", + "semver 1.0.22", ] [[package]] @@ -7380,9 +7361,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "safe_arith" @@ -7544,9 +7525,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" dependencies = [ "serde", ] @@ -7691,9 +7672,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.31" +version = "0.9.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adf8a49373e98a4c5f0ceb5d05aa7c648d75f63774981ed95b7c7443bbd50c6e" +checksum = "8fd075d994154d4a774f95b51fb96bdc2832b0ea48425c92546073816cda1f2f" dependencies = [ "indexmap 2.2.3", "itoa", @@ -7992,11 +7973,11 @@ dependencies = [ [[package]] name = "slog-term" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" dependencies = [ - "atty", + "is-terminal", "slog", "term", "thread_local", @@ -8526,9 +8507,7 @@ checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", - "libc", "num-conv", - "num_threads", "powerfmt", "serde", "time-core", diff --git a/Cargo.toml b/Cargo.toml index a7f44551ee8..317c3f95483 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,6 +94,7 @@ resolver = "2" edition = "2021" [workspace.dependencies] +anyhow = "1" arbitrary = { version = "1", features = ["derive"] } bincode = "1" bitvec = "1" diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index cd0de37d3ba..fe3e628f189 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -64,7 +64,7 @@ quick-protobuf-codec = "0.3" [dependencies.libp2p] version = "0.53" default-features = false -features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa", "metrics", "quic"] +features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa", "metrics", "quic", "upnp"] [dev-dependencies] slog-term = { workspace = true } diff --git a/beacon_node/lighthouse_network/src/service/behaviour.rs b/beacon_node/lighthouse_network/src/service/behaviour.rs index a43678d4ba3..5a04d6c2d84 100644 --- a/beacon_node/lighthouse_network/src/service/behaviour.rs +++ b/beacon_node/lighthouse_network/src/service/behaviour.rs @@ -6,6 +6,7 @@ use crate::types::SnappyTransform; use crate::gossipsub; use libp2p::identify; use libp2p::swarm::NetworkBehaviour; +use libp2p::upnp::tokio::Behaviour as Upnp; use types::EthSpec; use super::api_types::RequestId; @@ -32,6 +33,8 @@ where // NOTE: The id protocol is used for initial interop. This will be removed by mainnet. /// Provides IP addresses and peer information. pub identify: identify::Behaviour, + /// Libp2p UPnP port mapping. + pub upnp: Upnp, /// The routing pub-sub mechanism for eth2. pub gossipsub: Gossipsub, } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 401e43a53ff..3717c497386 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -28,10 +28,9 @@ use crate::{error, metrics, Enr, NetworkGlobals, PubsubMessage, TopicHash}; use api_types::{PeerRequestId, Request, RequestId, Response}; use futures::stream::StreamExt; use gossipsub_scoring_parameters::{lighthouse_gossip_thresholds, PeerScoreSettings}; -use libp2p::multiaddr::{Multiaddr, Protocol as MProtocol}; +use libp2p::multiaddr::{self, Multiaddr, Protocol as MProtocol}; use libp2p::swarm::{Swarm, SwarmEvent}; -use libp2p::PeerId; -use libp2p::{identify, SwarmBuilder}; +use libp2p::{identify, PeerId, SwarmBuilder}; use slog::{crit, debug, info, o, trace, warn}; use std::path::PathBuf; use std::pin::Pin; @@ -363,6 +362,7 @@ impl Network { identify, peer_manager, connection_limits, + upnp: Default::default(), } }; @@ -1601,6 +1601,47 @@ impl Network { } } + fn inject_upnp_event(&mut self, event: libp2p::upnp::Event) { + match event { + libp2p::upnp::Event::NewExternalAddr(addr) => { + info!(self.log, "UPnP route established"; "addr" => %addr); + let mut iter = addr.iter(); + // Skip Ip address. + iter.next(); + match iter.next() { + Some(multiaddr::Protocol::Udp(udp_port)) => match iter.next() { + Some(multiaddr::Protocol::QuicV1) => { + if let Err(e) = self.discovery_mut().update_enr_quic_port(udp_port) { + warn!(self.log, "Failed to update ENR"; "error" => e); + } + } + _ => { + trace!(self.log, "UPnP address mapped multiaddr from unknown transport"; "addr" => %addr) + } + }, + Some(multiaddr::Protocol::Tcp(tcp_port)) => { + if let Err(e) = self.discovery_mut().update_enr_tcp_port(tcp_port) { + warn!(self.log, "Failed to update ENR"; "error" => e); + } + } + _ => { + trace!(self.log, "UPnP address mapped multiaddr from unknown transport"; "addr" => %addr); + } + } + } + libp2p::upnp::Event::ExpiredExternalAddr(_) => {} + libp2p::upnp::Event::GatewayNotFound => { + info!(self.log, "UPnP not available"); + } + libp2p::upnp::Event::NonRoutableGateway => { + info!( + self.log, + "UPnP is available but gateway is not exposed to public network" + ); + } + } + } + /* Networking polling */ /// Poll the p2p networking stack. @@ -1623,6 +1664,10 @@ impl Network { } BehaviourEvent::Identify(ie) => self.inject_identify_event(ie), BehaviourEvent::PeerManager(pe) => self.inject_pm_event(pe), + BehaviourEvent::Upnp(e) => { + self.inject_upnp_event(e); + None + } BehaviourEvent::ConnectionLimits(le) => void::unreachable(le), }, SwarmEvent::ConnectionEstablished { .. } => None, diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 1dd6062edd4..489c5ae5293 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -8,7 +8,6 @@ use crate::{GossipTopic, NetworkConfig}; use futures::future::Either; use libp2p::core::{multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxed}; use libp2p::identity::{secp256k1, Keypair}; -use libp2p::quic; use libp2p::{core, noise, yamux, PeerId, Transport}; use prometheus_client::registry::Registry; use slog::{debug, warn}; @@ -63,8 +62,8 @@ pub fn build_transport( let transport = if quic_support { // Enables Quic // The default quic configuration suits us for now. - let quic_config = quic::Config::new(&local_private_key); - let quic = quic::tokio::Transport::new(quic_config); + let quic_config = libp2p::quic::Config::new(&local_private_key); + let quic = libp2p::quic::tokio::Transport::new(quic_config); let transport = tcp .or_transport(quic) .map(|either_output, _| match either_output { diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index d8766d0091e..58a195aea83 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -14,6 +14,7 @@ slog-async = { workspace = true } eth2 = { workspace = true } [dependencies] +anyhow = { workspace = true } beacon_chain = { workspace = true } store = { workspace = true } lighthouse_network = { workspace = true } @@ -35,11 +36,10 @@ lazy_static = { workspace = true } lighthouse_metrics = { workspace = true } logging = { workspace = true } task_executor = { workspace = true } -igd-next = "0.14.3" +igd-next = "0.14" itertools = { workspace = true } num_cpus = { workspace = true } lru_cache = { workspace = true } -if-addrs = "0.6.4" lru = { workspace = true } strum = { workspace = true } tokio-util = { workspace = true } @@ -56,4 +56,4 @@ environment = { workspace = true } # NOTE: This can be run via cargo build --bin lighthouse --features network/disable-backfill disable-backfill = [] fork_from_env = ["beacon_chain/fork_from_env"] -portable = ["beacon_chain/portable"] \ No newline at end of file +portable = ["beacon_chain/portable"] diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index cb81877b223..e63ff550398 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -3,231 +3,58 @@ //! Currently supported strategies: //! - UPnP -use crate::{NetworkConfig, NetworkMessage}; -use if_addrs::get_if_addrs; -use slog::{debug, info}; -use std::net::{IpAddr, SocketAddr, SocketAddrV4}; -use tokio::sync::mpsc; -use types::EthSpec; - -/// Configuration required to construct the UPnP port mappings. -pub struct UPnPConfig { - /// The local TCP port. - tcp_port: u16, - /// The local UDP discovery port. - disc_port: u16, - /// The local UDP quic port. - quic_port: u16, - /// Whether discovery is enabled or not. - disable_discovery: bool, - /// Whether quic is enabled or not. - disable_quic_support: bool, -} - -/// Contains mappings that managed to be established. -#[derive(Default, Debug)] -pub struct EstablishedUPnPMappings { - /// A TCP port mapping for libp2p. - pub tcp_port: Option, - /// A UDP port for the QUIC libp2p transport. - pub udp_quic_port: Option, - /// A UDP port for discv5. - pub udp_disc_port: Option, -} - -impl EstablishedUPnPMappings { - /// Returns true if at least one value is set. - pub fn is_some(&self) -> bool { - self.tcp_port.is_some() || self.udp_quic_port.is_some() || self.udp_disc_port.is_some() - } - - // Iterator over the UDP ports - pub fn udp_ports(&self) -> impl Iterator { - self.udp_quic_port.iter().chain(self.udp_disc_port.iter()) - } -} - -impl UPnPConfig { - pub fn from_config(config: &NetworkConfig) -> Option { - config.listen_addrs().v4().map(|v4_addr| UPnPConfig { - tcp_port: v4_addr.tcp_port, - disc_port: v4_addr.disc_port, - quic_port: v4_addr.quic_port, - disable_discovery: config.disable_discovery, - disable_quic_support: config.disable_quic_support, - }) - } -} - -/// Attempts to construct external port mappings with UPnP. -pub fn construct_upnp_mappings( - config: UPnPConfig, - network_send: mpsc::UnboundedSender>, +use anyhow::{bail, Context, Error}; +use igd_next::{aio::tokio as igd, PortMappingProtocol}; +use slog::debug; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::time::Duration; +use tokio::time::sleep; + +/// The duration in seconds of a port mapping on the gateway. +const MAPPING_DURATION: u32 = 3600; + +/// Renew the Mapping every half of `MAPPING_DURATION` to avoid the port being unmapped. +const MAPPING_TIMEOUT: u64 = MAPPING_DURATION as u64 / 2; + +/// Attempts to map Discovery external port mappings with UPnP. +pub async fn construct_upnp_mappings( + addr: Ipv4Addr, + port: u16, log: slog::Logger, -) { - info!(log, "UPnP Attempting to initialise routes"); - match igd_next::search_gateway(Default::default()) { - Err(e) => info!(log, "UPnP not available"; "error" => %e), - Ok(gateway) => { - // Need to find the local listening address matched with the router subnet - let interfaces = match get_if_addrs() { - Ok(v) => v, - Err(e) => { - info!(log, "UPnP failed to get local interfaces"; "error" => %e); - return; - } - }; - let local_ip = interfaces.iter().find_map(|interface| { - // Just use the first IP of the first interface that is not a loopback and not an - // ipv6 address. - if !interface.is_loopback() { - interface.ip().is_ipv4().then(|| interface.ip()) - } else { - None - } - }); - - let local_ip = match local_ip { - None => { - info!(log, "UPnP failed to find local IP address"); - return; - } - Some(v) => v, - }; - - debug!(log, "UPnP Local IP Discovered"; "ip" => ?local_ip); - - let mut mappings = EstablishedUPnPMappings::default(); - - match local_ip { - IpAddr::V4(address) => { - let libp2p_socket = SocketAddrV4::new(address, config.tcp_port); - let external_ip = gateway.get_external_ip(); - // We add specific port mappings rather than getting the router to arbitrary assign - // one. - // I've found this to be more reliable. If multiple users are behind a single - // router, they should ideally try to set different port numbers. - mappings.tcp_port = add_port_mapping( - &gateway, - igd_next::PortMappingProtocol::TCP, - libp2p_socket, - "tcp", - &log, - ).map(|_| { - let external_socket = external_ip.as_ref().map(|ip| SocketAddr::new(*ip, config.tcp_port)).map_err(|_| ()); - info!(log, "UPnP TCP route established"; "external_socket" => format!("{}:{}", external_socket.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.tcp_port)); - config.tcp_port - }).ok(); - - let set_udp_mapping = |udp_port| { - let udp_socket = SocketAddrV4::new(address, udp_port); - add_port_mapping( - &gateway, - igd_next::PortMappingProtocol::UDP, - udp_socket, - "udp", - &log, - ).map(|_| { - info!(log, "UPnP UDP route established"; "external_socket" => format!("{}:{}", external_ip.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), udp_port)); - }) - }; - - // Set the discovery UDP port mapping - if !config.disable_discovery && set_udp_mapping(config.disc_port).is_ok() { - mappings.udp_disc_port = Some(config.disc_port); - } - - // Set the quic UDP port mapping - if !config.disable_quic_support && set_udp_mapping(config.quic_port).is_ok() { - mappings.udp_quic_port = Some(config.quic_port); - } - - // report any updates to the network service. - if mappings.is_some() { - network_send.send(NetworkMessage::UPnPMappingEstablished{ mappings }) - .unwrap_or_else(|e| debug!(log, "Could not send message to the network service"; "error" => %e)); - } - } - _ => debug!(log, "UPnP no routes constructed. IPv6 not supported"), - } - } +) -> Result<(), Error> { + let gateway = igd::search_gateway(Default::default()) + .await + .context("Gateway does not support UPnP")?; + + let external_address = gateway + .get_external_ip() + .await + .context("Could not access gateway's external ip")?; + + let is_private = match external_address { + IpAddr::V4(ipv4) => ipv4.is_private(), + IpAddr::V6(ipv6) => ipv6.is_loopback() || ipv6.is_unspecified(), }; -} -/// Sets up a port mapping for a protocol returning the mapped port if successful. -fn add_port_mapping( - gateway: &igd_next::Gateway, - protocol: igd_next::PortMappingProtocol, - socket: SocketAddrV4, - protocol_string: &'static str, - log: &slog::Logger, -) -> Result<(), ()> { - // We add specific port mappings rather than getting the router to arbitrary assign - // one. - // I've found this to be more reliable. If multiple users are behind a single - // router, they should ideally try to set different port numbers. - let mapping_string = &format!("lighthouse-{}", protocol_string); - for _ in 0..2 { - match gateway.add_port( - protocol, - socket.port(), - SocketAddr::V4(socket), - 0, - mapping_string, - ) { - Err(e) => { - match e { - igd_next::AddPortError::PortInUse => { - // Try and remove and re-create - debug!(log, "UPnP port in use, attempting to remap"; "protocol" => protocol_string, "port" => socket.port()); - match gateway.remove_port(protocol, socket.port()) { - Ok(()) => { - debug!(log, "UPnP Removed port mapping"; "protocol" => protocol_string, "port" => socket.port()) - } - Err(e) => { - debug!(log, "UPnP Port remove failure"; "protocol" => protocol_string, "port" => socket.port(), "error" => %e); - return Err(()); - } - } - } - e => { - info!(log, "UPnP TCP route not set"; "error" => %e); - return Err(()); - } - } - } - Ok(_) => { - return Ok(()); - } - } + if is_private { + bail!( + "Gateway's external address is a private address: {}", + external_address + ); } - Err(()) -} -/// Removes the specified TCP and UDP port mappings. -pub fn remove_mappings(mappings: &EstablishedUPnPMappings, log: &slog::Logger) { - if mappings.is_some() { - debug!(log, "Removing UPnP port mappings"); - match igd_next::search_gateway(Default::default()) { - Ok(gateway) => { - if let Some(tcp_port) = mappings.tcp_port { - match gateway.remove_port(igd_next::PortMappingProtocol::TCP, tcp_port) { - Ok(()) => debug!(log, "UPnP Removed TCP port mapping"; "port" => tcp_port), - Err(e) => { - debug!(log, "UPnP Failed to remove TCP port mapping"; "port" => tcp_port, "error" => %e) - } - } - } - for udp_port in mappings.udp_ports() { - match gateway.remove_port(igd_next::PortMappingProtocol::UDP, *udp_port) { - Ok(()) => debug!(log, "UPnP Removed UDP port mapping"; "port" => udp_port), - Err(e) => { - debug!(log, "UPnP Failed to remove UDP port mapping"; "port" => udp_port, "error" => %e) - } - } - } - } - Err(e) => debug!(log, "UPnP failed to remove mappings"; "error" => %e), - } + loop { + gateway + .add_port( + PortMappingProtocol::UDP, + port, + SocketAddr::new(IpAddr::V4(addr), port), + MAPPING_DURATION, + "Lighthouse Discovery port", + ) + .await + .with_context(|| format!("Could not UPnP map port: {} on the gateway", port))?; + debug!(log, "Discovery UPnP port mapped"; "port" => %port); + sleep(Duration::from_secs(MAPPING_TIMEOUT)).await; } } diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 01a7e1f9896..0905f4a07b5 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -1,5 +1,5 @@ use super::sync::manager::RequestId as SyncId; -use crate::nat::EstablishedUPnPMappings; +use crate::nat; use crate::network_beacon_processor::InvalidBlockStorage; use crate::persisted_dht::{clear_dht, load_dht, persist_dht}; use crate::router::{Router, RouterMessage}; @@ -94,11 +94,6 @@ pub enum NetworkMessage { /// The result of the validation validation_result: MessageAcceptance, }, - /// Called if UPnP managed to establish an external port mapping. - UPnPMappingEstablished { - /// The mappings that were established. - mappings: EstablishedUPnPMappings, - }, /// Reports a peer to the peer manager for performing an action. ReportPeer { peer_id: PeerId, @@ -188,9 +183,6 @@ pub struct NetworkService { store: Arc>, /// A collection of global variables, accessible outside of the network service. network_globals: Arc>, - /// Stores potentially created UPnP mappings to be removed on shutdown. (TCP port and UDP - /// ports). - upnp_mappings: EstablishedUPnPMappings, /// A delay that expires when a new fork takes place. next_fork_update: Pin>>, /// A delay that expires when we need to subscribe to a new fork's topics. @@ -237,22 +229,24 @@ impl NetworkService { "Backfill is disabled. DO NOT RUN IN PRODUCTION" ); - // try and construct UPnP port mappings if required. - if let Some(upnp_config) = crate::nat::UPnPConfig::from_config(config) { - let upnp_log = network_log.new(o!("service" => "UPnP")); - let upnp_network_send = network_senders.network_send(); - if config.upnp_enabled { - executor.spawn_blocking( - move || { - crate::nat::construct_upnp_mappings( - upnp_config, - upnp_network_send, - upnp_log, - ) - }, - "UPnP", - ); - } + if let (true, false, Some(v4)) = ( + config.upnp_enabled, + config.disable_discovery, + config.listen_addrs().v4(), + ) { + let nw = network_log.clone(); + let v4 = v4.clone(); + executor.spawn( + async move { + info!(nw, "UPnP Attempting to initialise routes"); + if let Err(e) = + nat::construct_upnp_mappings(v4.addr, v4.disc_port, nw.clone()).await + { + info!(nw, "Could not UPnP map Discovery port"; "error" => %e); + } + }, + "UPnP", + ); } // get a reference to the beacon chain store @@ -358,7 +352,6 @@ impl NetworkService { router_send, store, network_globals: network_globals.clone(), - upnp_mappings: EstablishedUPnPMappings::default(), next_fork_update, next_fork_subscriptions, next_unsubscribe, @@ -636,21 +629,6 @@ impl NetworkService { } => { self.libp2p.send_error_response(peer_id, id, error, reason); } - NetworkMessage::UPnPMappingEstablished { mappings } => { - self.upnp_mappings = mappings; - // If there is an external TCP port update, modify our local ENR. - if let Some(tcp_port) = self.upnp_mappings.tcp_port { - if let Err(e) = self.libp2p.discovery_mut().update_enr_tcp_port(tcp_port) { - warn!(self.log, "Failed to update ENR"; "error" => e); - } - } - // If there is an external QUIC port update, modify our local ENR. - if let Some(quic_port) = self.upnp_mappings.udp_quic_port { - if let Err(e) = self.libp2p.discovery_mut().update_enr_quic_port(quic_port) { - warn!(self.log, "Failed to update ENR"; "error" => e); - } - } - } NetworkMessage::ValidationResult { propagation_source, message_id, @@ -1009,10 +987,6 @@ impl Drop for NetworkService { "Saved DHT state"; ), } - - // attempt to remove port mappings - crate::nat::remove_mappings(&self.upnp_mappings, &self.log); - info!(self.log, "Network service shutdown"); } }