Merge branch 'unstable' into vm-voluntary-exit

.github/workflows/docker.yml

@@ -13,8 +13,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-    DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    DOCKER_PASSWORD: ${{ secrets.DH_KEY }}
+    DOCKER_USERNAME: ${{ secrets.DH_ORG }}
     # Enable self-hosted runners for the sigp repo only.
     SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
 

.github/workflows/release.yml

@@ -10,8 +10,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-    DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    DOCKER_PASSWORD: ${{ secrets.DH_KEY }}
+    DOCKER_USERNAME: ${{ secrets.DH_ORG }}
     REPO_NAME: ${{ github.repository_owner }}/lighthouse
     IMAGE_NAME: ${{ github.repository_owner }}/lighthouse
     # Enable self-hosted runners for the sigp repo only.

Cargo.toml

@@ -201,7 +201,7 @@ tree_hash_derive = "0.8"
 url = "2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 warp = { version = "0.3.7", default-features = false, features = ["tls"] }
-zeroize = { version = "1", features = ["zeroize_derive"] }
+zeroize = { version = "1", features = ["zeroize_derive", "serde"] }
 zip = "0.6"
 
 # Local crates.

Makefile

@@ -204,7 +204,7 @@ test-full: cargo-fmt test-release test-debug test-ef test-exec-engine
 # Lints the code for bad style and potentially unsafe arithmetic using Clippy.
 # Clippy lints are opt-in per-crate for now. By default, everything is allowed except for performance and correctness lints.
 lint:
-	cargo clippy --workspace --benches --tests $(EXTRA_CLIPPY_OPTS) --features "$(TEST_FEATURES)" -- \
+	RUSTFLAGS="-C debug-assertions=no $(RUSTFLAGS)" cargo clippy --workspace --benches --tests $(EXTRA_CLIPPY_OPTS) --features "$(TEST_FEATURES)" -- \
 		-D clippy::fn_to_numeric_cast_any \
 		-D clippy::manual_let_else \
 		-D clippy::large_stack_frames \

account_manager/Cargo.toml

@@ -27,6 +27,7 @@ safe_arith = { workspace = true }
 slot_clock = { workspace = true }
 filesystem = { workspace = true }
 sensitive_url = { workspace = true }
+zeroize = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }

account_manager/src/validator/create.rs

@@ -294,7 +294,7 @@ pub fn read_wallet_password_from_cli(
             eprintln!();
             eprintln!("{}", WALLET_PASSWORD_PROMPT);
             let password =
-                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_bytes().to_vec());
             Ok(password)
         }
     }

account_manager/src/validator/exit.rs

@@ -409,6 +409,6 @@ mod tests {
         )
         .unwrap();
 
-        assert_eq!(expected_pk, kp.pk.into());
+        assert_eq!(expected_pk, kp.pk);
     }
 }

account_manager/src/validator/import.rs

@@ -7,7 +7,7 @@ use account_utils::{
         recursively_find_voting_keystores, PasswordStorage, ValidatorDefinition,
         ValidatorDefinitions, CONFIG_FILENAME,
     },
-    ZeroizeString, STDIN_INPUTS_FLAG,
+    STDIN_INPUTS_FLAG,
 };
 use clap::{Arg, ArgAction, ArgMatches, Command};
 use clap_utils::FLAG_HEADER;
@@ -16,6 +16,7 @@ use std::fs;
 use std::path::PathBuf;
 use std::thread::sleep;
 use std::time::Duration;
+use zeroize::Zeroizing;
 
 pub const CMD: &str = "import";
 pub const KEYSTORE_FLAG: &str = "keystore";
@@ -148,7 +149,7 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
     // Skip keystores that already exist, but exit early if any operation fails.
     // Reuses the same password for all keystores if the `REUSE_PASSWORD_FLAG` flag is set.
     let mut num_imported_keystores = 0;
-    let mut previous_password: Option<ZeroizeString> = None;
+    let mut previous_password: Option<Zeroizing<String>> = None;
 
     for src_keystore in &keystore_paths {
         let keystore = Keystore::from_json_file(src_keystore)
@@ -182,14 +183,17 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
 
             let password = match keystore_password_path.as_ref() {
                 Some(path) => {
-                    let password_from_file: ZeroizeString = fs::read_to_string(path)
+                    let password_from_file: Zeroizing<String> = fs::read_to_string(path)
                         .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))?
                         .into();
-                    password_from_file.without_newlines()
+                    password_from_file
+                        .trim_end_matches(['\r', '\n'])
+                        .to_string()
+                        .into()
                 }
                 None => {
                     let password_from_user = read_password_from_user(stdin_inputs)?;
-                    if password_from_user.as_ref().is_empty() {
+                    if password_from_user.is_empty() {
                         eprintln!("Continuing without password.");
                         sleep(Duration::from_secs(1)); // Provides nicer UX.
                         break None;
@@ -314,7 +318,7 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
 /// Otherwise, returns the keystore error.
 fn check_password_on_keystore(
     keystore: &Keystore,
-    password: &ZeroizeString,
+    password: &Zeroizing<String>,
 ) -> Result<bool, String> {
     match keystore.decrypt_keypair(password.as_ref()) {
         Ok(_) => {

account_manager/src/wallet/create.rs

@@ -226,14 +226,14 @@ pub fn read_new_wallet_password_from_cli(
             eprintln!();
             eprintln!("{}", NEW_WALLET_PASSWORD_PROMPT);
             let password =
-                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_bytes().to_vec());
 
             // Ensure the password meets the minimum requirements.
             match is_password_sufficiently_complex(password.as_bytes()) {
                 Ok(_) => {
                     eprintln!("{}", RETYPE_PASSWORD_PROMPT);
                     let retyped_password =
-                        PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+                        PlainText::from(read_password_from_user(stdin_inputs)?.as_bytes().to_vec());
                     if retyped_password == password {
                         break Ok(password);
                     } else {

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "6.0.0"
+version = "6.0.1"
 authors = [
     "Paul Hauner <paul@paulhauner.com>",
     "Age Manning <Age@AgeManning.com",

beacon_node/beacon_chain/src/block_verification.rs

@@ -92,6 +92,7 @@ use std::fs;
 use std::io::Write;
 use std::sync::Arc;
 use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp};
+use strum::AsRefStr;
 use task_executor::JoinHandle;
 use types::{
     data_column_sidecar::DataColumnSidecarError, BeaconBlockRef, BeaconState, BeaconStateError,
@@ -137,7 +138,7 @@ const WRITE_BLOCK_PROCESSING_SSZ: bool = cfg!(feature = "write_ssz_files");
 ///
 /// - The block is malformed/invalid (indicated by all results other than `BeaconChainError`.
 /// - We encountered an error whilst trying to verify the block (a `BeaconChainError`).
-#[derive(Debug)]
+#[derive(Debug, AsRefStr)]
 pub enum BlockError {
     /// The parent block was unknown.
     ///

beacon_node/beacon_chain/src/builder.rs

@@ -1037,7 +1037,9 @@ where
         );
 
         // Check for states to reconstruct (in the background).
-        if beacon_chain.config.reconstruct_historic_states {
+        if beacon_chain.config.reconstruct_historic_states
+            && beacon_chain.store.get_oldest_block_slot() == 0
+        {
             beacon_chain.store_migrator.process_reconstruction();
         }
 

beacon_node/beacon_chain/src/execution_payload.rs

@@ -14,7 +14,7 @@ use crate::{
 };
 use execution_layer::{
     BlockProposalContents, BlockProposalContentsType, BuilderParams, NewPayloadRequest,
-    PayloadAttributes, PayloadStatus,
+    PayloadAttributes, PayloadParameters, PayloadStatus,
 };
 use fork_choice::{InvalidationOperation, PayloadVerificationStatus};
 use proto_array::{Block as ProtoBlock, ExecutionStatus};
@@ -375,8 +375,9 @@ pub fn get_execution_payload<T: BeaconChainTypes>(
     let timestamp =
         compute_timestamp_at_slot(state, state.slot(), spec).map_err(BeaconStateError::from)?;
     let random = *state.get_randao_mix(current_epoch)?;
-    let latest_execution_payload_header_block_hash =
-        state.latest_execution_payload_header()?.block_hash();
+    let latest_execution_payload_header = state.latest_execution_payload_header()?;
+    let latest_execution_payload_header_block_hash = latest_execution_payload_header.block_hash();
+    let latest_execution_payload_header_gas_limit = latest_execution_payload_header.gas_limit();
     let withdrawals = match state {
         &BeaconState::Capella(_) | &BeaconState::Deneb(_) | &BeaconState::Electra(_) => {
             Some(get_expected_withdrawals(state, spec)?.0.into())
@@ -406,6 +407,7 @@ pub fn get_execution_payload<T: BeaconChainTypes>(
                     random,
                     proposer_index,
                     latest_execution_payload_header_block_hash,
+                    latest_execution_payload_header_gas_limit,
                     builder_params,
                     withdrawals,
                     parent_beacon_block_root,
@@ -443,6 +445,7 @@ pub async fn prepare_execution_payload<T>(
     random: Hash256,
     proposer_index: u64,
     latest_execution_payload_header_block_hash: ExecutionBlockHash,
+    latest_execution_payload_header_gas_limit: u64,
     builder_params: BuilderParams,
     withdrawals: Option<Vec<Withdrawal>>,
     parent_beacon_block_root: Option<Hash256>,
@@ -526,13 +529,20 @@ where
         parent_beacon_block_root,
     );
 
+    let target_gas_limit = execution_layer.get_proposer_gas_limit(proposer_index).await;
+    let payload_parameters = PayloadParameters {
+        parent_hash,
+        parent_gas_limit: latest_execution_payload_header_gas_limit,
+        proposer_gas_limit: target_gas_limit,
+        payload_attributes: &payload_attributes,
+        forkchoice_update_params: &forkchoice_update_params,
+        current_fork: fork,
+    };
+
     let block_contents = execution_layer
         .get_payload(
-            parent_hash,
-            &payload_attributes,
-            forkchoice_update_params,
+            payload_parameters,
             builder_params,
-            fork,
             &chain.spec,
             builder_boost_factor,
             block_production_version,

beacon_node/beacon_chain/src/migrate.rs

@@ -26,8 +26,10 @@ const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
 const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
 /// Maximum number of blocks applied in each reconstruction burst.
 ///
-/// This limits the amount of time that the finalization migration is paused for.
-const BLOCKS_PER_RECONSTRUCTION: usize = 8192 * 4;
+/// This limits the amount of time that the finalization migration is paused for. We set this
+/// conservatively because pausing the finalization migration for too long can cause hot state
+/// cache misses and excessive disk use.
+const BLOCKS_PER_RECONSTRUCTION: usize = 1024;
 
 /// Default number of epochs to wait between finalization migrations.
 pub const DEFAULT_EPOCHS_PER_MIGRATION: u64 = 1;

beacon_node/beacon_chain/src/schema_change/migration_schema_v22.rs

@@ -152,7 +152,7 @@ pub fn delete_old_schema_freezer_data<T: BeaconChainTypes>(
     db.cold_db.do_atomically(cold_ops)?;
 
     // In order to reclaim space, we need to compact the freezer DB as well.
-    db.cold_db.compact()?;
+    db.compact_freezer()?;
 
     Ok(())
 }

beacon_node/beacon_chain/src/shuffling_cache.rs

@@ -512,7 +512,7 @@ mod test {
         }
 
         assert!(
-            !cache.contains(&shuffling_id_and_committee_caches.get(0).unwrap().0),
+            !cache.contains(&shuffling_id_and_committee_caches.first().unwrap().0),
             "should not contain oldest epoch shuffling id"
         );
         assert_eq!(