From 557d1782b20977513f106c4d7776a2e30c2d560c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 18 Sep 2024 18:43:46 +0000 Subject: [PATCH 001/241] Integrate config tool into setupOS --- ic-os/components/misc/config.sh | 12 ++ .../setupos-scripts/check-config.sh | 33 ++++ .../setupos-scripts/check-hardware.sh | 11 +- .../setupos-scripts/check-network.sh | 66 +++----- .../components/setupos-scripts/config.service | 6 +- ic-os/components/setupos-scripts/config.sh | 120 -------------- ic-os/components/setupos-scripts/functions.sh | 2 +- .../setupos-scripts/install-guestos.sh | 3 +- .../setupos-scripts/install-hostos.sh | 3 +- .../components/setupos-scripts/setup-disk.sh | 3 +- .../setupos-scripts/setup-hostos-config.sh | 44 +++-- ic-os/components/setupos-scripts/setupos.sh | 3 +- ic-os/components/setupos.bzl | 4 +- ic-os/setupos/defs.bzl | 1 + rs/ic_os/network/src/info.rs | 154 ------------------ rs/ic_os/network/src/interfaces.rs | 4 +- rs/ic_os/network/src/lib.rs | 25 ++- rs/ic_os/network/src/systemd.rs | 21 ++- rs/ic_os/os_tools/hostos_tool/src/main.rs | 126 ++++++++------ rs/ic_os/os_tools/setupos_tool/src/main.rs | 95 +++++------ 20 files changed, 270 insertions(+), 466 deletions(-) create mode 100644 ic-os/components/misc/config.sh create mode 100644 ic-os/components/setupos-scripts/check-config.sh delete mode 100755 ic-os/components/setupos-scripts/config.sh delete mode 100644 rs/ic_os/network/src/info.rs diff --git a/ic-os/components/misc/config.sh b/ic-os/components/misc/config.sh new file mode 100644 index 00000000000..21151813cc8 --- /dev/null +++ b/ic-os/components/misc/config.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Shared config utilities. + +# Retrieves a value from the config.json file using a JSON path. +# Arguments: +# $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') +function get_config_value() { + local CONFIG_FILE="/var/ic/config/config.json" + local key=$1 + jq -r "${key}" "${CONFIG_FILE}" +} diff --git a/ic-os/components/setupos-scripts/check-config.sh b/ic-os/components/setupos-scripts/check-config.sh new file mode 100644 index 00000000000..b826345a4c5 --- /dev/null +++ b/ic-os/components/setupos-scripts/check-config.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o nounset +set -o pipefail + +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + +SHELL="/bin/bash" +PATH="/sbin:/bin:/usr/sbin:/usr/bin" + +check_config_file() { + echo "* Checking Config..." + local CONFIG_FILE="/var/ic/config/config.json" + + if [ -f "${CONFIG_FILE}" ]; then + local config_contents=$(cat "${CONFIG_FILE}") + echo -e "Configuration file '${CONFIG_FILE}' exists.\n" + echo -e "File contents:\n${config_contents}" + else + local service_logs=$(journalctl -u config.service --no-pager) + local log_message="Error creating SetupOS configuration. Configuration file '${CONFIG_FILE}' does not exist.\n\nConfig.service logs:\n${service_logs}" + + log_and_halt_installation_on_error 1 "${log_message}" + fi +} + +# Establish run order +main() { + check_config_file +} + +main diff --git a/ic-os/components/setupos-scripts/check-hardware.sh b/ic-os/components/setupos-scripts/check-hardware.sh index 97770a9d8bd..3d77b7ac2cb 100644 --- a/ic-os/components/setupos-scripts/check-hardware.sh +++ b/ic-os/components/setupos-scripts/check-hardware.sh @@ -3,6 +3,9 @@ set -o nounset set -o pipefail +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" @@ -30,8 +33,6 @@ GEN2_MINIMUM_AGGREGATE_DISK_SIZE=32000000000000 GEN1_MINIMUM_DISK_SIZE=3200000000000 GEN1_MINIMUM_AGGREGATE_DISK_SIZE=32000000000000 -CONFIG_DIR="/var/ic/config" - function check_generation() { echo "* Checking Generation..." @@ -247,7 +248,10 @@ function verify_disks() { function verify_deployment_path() { echo "* Verifying deployment path..." - if [[ ${GENERATION} == 2 ]] && [[ ! -f "${CONFIG_DIR}/node_operator_private_key.pem" ]]; then + + local node_operator_key_path=$(get_config_value '.icos_settings.node_operator_private_key_path') + + if [[ ${GENERATION} == 2 ]] && [[ ! -f "${node_operator_key_path}" ]]; then echo -e "\n\n\n\n\n\n" echo -e "\033[1;31mWARNING: Gen2 hardware detected but no Node Operator Private Key found.\033[0m" echo -e "\033[1;31mGen2 hardware should be deployed using the Gen2 Node Deployment method.\033[0m" @@ -261,7 +265,6 @@ function verify_deployment_path() { # Establish run order main() { - source /opt/ic/bin/functions.sh log_start "$(basename $0)" check_generation verify_cpu diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index 057a9ef56dd..8eb9932a420 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -3,25 +3,19 @@ set -o nounset set -o pipefail +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -CONFIG="${CONFIG:=/var/ic/config/config.ini}" -DEPLOYMENT="${DEPLOYMENT:=/data/deployment.json}" - -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; - "ipv4_address") ipv4_address="${value}" ;; - "ipv4_prefix_length") ipv4_prefix_length="${value}" ;; - "ipv4_gateway") ipv4_gateway="${value}" ;; - "domain") domain="${value}" ;; - esac - done <"${CONFIG}" +function read_config_variables() { + ipv6_prefix=$(get_config_value '.network_settings.ipv6_prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_gateway') + domain=$(get_config_value '.network_settings.domain') } # WARNING: Uses 'eval' for command execution. @@ -168,45 +162,36 @@ function ping_ipv6_gateway() { echo " " } -function assemble_nns_nodes_list() { - NNS_URL_STRING=$(/opt/ic/bin/fetch-property.sh --key=.nns.url --config=${DEPLOYMENT}) - NNS_URL_LIST=$(echo $NNS_URL_STRING | sed 's@,@ @g') -} - function query_nns_nodes() { echo "* Querying NNS nodes..." - i=0 - success=0 - nodes=$(echo ${NNS_URL_LIST} | wc -w) - # At least one of the provided URLs needs to work. - verify=1 - for url in $(echo $NNS_URL_LIST); do + local nns_urls=($(get_config_value '.icos_settings.nns_urls' | jq -r '.[]')) + local success=false + + for url in "${nns_urls[@]}"; do # When running against testnets, we need to ignore self signed certs # with `--insecure`. This check is only meant to confirm from SetupOS # that NNS urls are reachable, so we do not mind that it is "weak". - curl --insecure --head --connect-timeout 3 --silent ${url} >/dev/null 2>&1 - if [ "${?}" -ne 0 ]; then - echo " fail: ${url}" - else + if curl --insecure --head --connect-timeout 3 --silent "${url}" >/dev/null 2>&1; then echo " okay: ${url}" - success=$((${success} + 1)) - fi - i=$((${i} + 1)) - if [ ${success} -ge ${verify} ]; then - echo " success" + success=true break - elif [ ${i} -eq ${nodes} ]; then - log_and_halt_installation_on_error "1" "Unable to query enough healthy NNS nodes." + else + echo " fail: ${url}" fi done + + if $success; then + echo " success" + else + log_and_halt_installation_on_error "1" "Unable to query enough healthy NNS nodes." + fi } # Establish run order main() { - source /opt/ic/bin/functions.sh log_start "$(basename $0)" - read_variables + read_config_variables get_network_settings print_network_settings @@ -217,7 +202,6 @@ main() { fi ping_ipv6_gateway - assemble_nns_nodes_list query_nns_nodes log_end "$(basename $0)" } diff --git a/ic-os/components/setupos-scripts/config.service b/ic-os/components/setupos-scripts/config.service index 2db6cdf7dc0..f35bbae5eb3 100644 --- a/ic-os/components/setupos-scripts/config.service +++ b/ic-os/components/setupos-scripts/config.service @@ -6,9 +6,9 @@ Before=setupos.service [Service] Type=oneshot RemainAfterExit=true -ExecStart=/opt/ic/bin/output-wrapper.sh /dev/ttyS0 /opt/ic/bin/config.sh -StandardOutput=tty -StandardError=tty +ExecStart=/opt/ic/bin/config create-setupos-config +StandardOutput=journal+console +StandardError=journal+console [Install] WantedBy=multi-user.target diff --git a/ic-os/components/setupos-scripts/config.sh b/ic-os/components/setupos-scripts/config.sh deleted file mode 100755 index 0b13fb29296..00000000000 --- a/ic-os/components/setupos-scripts/config.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o pipefail - -SHELL="/bin/bash" -PATH="/sbin:/bin:/usr/sbin:/usr/bin" - -CONFIG_DIR="/config" -CONFIG_TMP="/var/ic/config" -CONFIG_INI="${CONFIG_DIR}/config.ini" -CONFIG_INI_CLONE="${CONFIG_TMP}/config.ini" -SSH_AUTHORIZED_KEYS="${CONFIG_DIR}/ssh_authorized_keys" -SSH_AUTHORIZED_KEYS_CLONE="${CONFIG_TMP}/ssh_authorized_keys" - -# Define empty variables so they are not unset -ipv6_prefix="" -ipv6_gateway="" - -function print_config_file() { - if [ -e "${CONFIG_INI}" ]; then - echo "Found ${CONFIG_INI}. Contents:" - cat "${CONFIG_INI}" - else - log_and_halt_installation_on_error "1" "config.ini not found. Please copy a valid config.ini to the SetupOS installer config partition." - fi - -} - -function create_config_tmp() { - if [ ! -e "${CONFIG_TMP}" ]; then - # Create fresh config tmp directory - mkdir -p "${CONFIG_TMP}" - log_and_halt_installation_on_error "${?}" "Unable to create new '${CONFIG_TMP}' directory." - fi -} - -function clone_config() { - cp "${CONFIG_INI}" "${CONFIG_INI_CLONE}" - log_and_halt_installation_on_error "${?}" "Unable to copy 'config.ini' configuration file." - - if [ ! -f "${CONFIG_INI_CLONE}" ]; then - log_and_halt_installation_on_error "1" "Cloned 'config.ini' configuration file does not exist." - fi - - if [ -f "${CONFIG_DIR}/node_operator_private_key.pem" ]; then - cp ${CONFIG_DIR}/node_operator_private_key.pem ${CONFIG_TMP}/node_operator_private_key.pem - log_and_halt_installation_on_error "${?}" "Unable to copy 'node_operator_private_key.pem' configuration file." - fi - - if [ -d "${SSH_AUTHORIZED_KEYS}" ]; then - cp -r "${SSH_AUTHORIZED_KEYS}" "${CONFIG_TMP}" - log_and_halt_installation_on_error "${?}" "Unable to copy 'ssh_authorized_keys' directory." - else - log_and_halt_installation_on_error "1" "Unable to read 'ssh_authorized_keys' directory." - fi - - if [ ! -d "${SSH_AUTHORIZED_KEYS_CLONE}" ]; then - log_and_halt_installation_on_error "1" "Cloned 'ssh_authorized_keys' directory does not exist." - fi -} - -function normalize_config() { - CONFIG_VAR=$(cat "${CONFIG_INI_CLONE}" | tr '\r' '\n') - echo "${CONFIG_VAR}" >"${CONFIG_INI_CLONE}" - - sed -i 's/#.*$//g' "${CONFIG_INI_CLONE}" - log_and_halt_installation_on_error "${?}" "Unable to remove comments from 'config.ini'." - - sed -i 's/"//g' "${CONFIG_INI_CLONE}" - log_and_halt_installation_on_error "${?}" "Unable to replace double-quote characters in 'config.ini'." - - sed -i "s/'//g" "${CONFIG_INI_CLONE}" - log_and_halt_installation_on_error "${?}" "Unable to replace single-quote characters in 'config.ini'." - - sed -i 's/.*/\L&/' "${CONFIG_INI_CLONE}" - log_and_halt_installation_on_error "${?}" "Unable to convert upper- to lower-case in 'config.ini'." - - sed -i '/^$/d' "${CONFIG_INI_CLONE}" - log_and_halt_installation_on_error "${?}" "Unable to remove empty lines in 'config.ini'." - - echo -e '\n' >>"${CONFIG_INI_CLONE}" - log_and_halt_installation_on_error "${?}" "Unable to inject extra new-line at the end of 'config.ini'." -} - -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; - esac - done <"${CONFIG_INI_CLONE}" -} - -function verify_variables() { - if [ -z "${ipv6_prefix}" ]; then - log_and_halt_installation_on_error "1" "Variable 'ipv6_prefix' is not defined in 'config.ini'." - fi - - if [ -z "${ipv6_gateway}" ]; then - log_and_halt_installation_on_error "1" "Variable 'ipv6_gateway' is not defined in 'config.ini'." - fi -} - -# Establish run order -main() { - source /opt/ic/bin/functions.sh - log_start "$(basename $0)" - print_config_file - create_config_tmp - clone_config - normalize_config - read_variables - verify_variables - log_end "$(basename $0)" -} - -main diff --git a/ic-os/components/setupos-scripts/functions.sh b/ic-os/components/setupos-scripts/functions.sh index c058b1885c2..4e63bf0782e 100755 --- a/ic-os/components/setupos-scripts/functions.sh +++ b/ic-os/components/setupos-scripts/functions.sh @@ -23,7 +23,7 @@ function log_and_halt_installation_on_error() { echo " ERROR" echo "--------------------------------------------------------------------------------" echo -e "\n\n" - echo "${log_message}" + echo -e "${log_message}" echo -e "\n\n" echo "--------------------------------------------------------------------------------" echo " ERROR" diff --git a/ic-os/components/setupos-scripts/install-guestos.sh b/ic-os/components/setupos-scripts/install-guestos.sh index 2311c3ba9c3..3e24773234d 100755 --- a/ic-os/components/setupos-scripts/install-guestos.sh +++ b/ic-os/components/setupos-scripts/install-guestos.sh @@ -3,6 +3,8 @@ set -o nounset set -o pipefail +source /opt/ic/bin/functions.sh + SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" @@ -34,7 +36,6 @@ function install_guestos() { # Establish run order main() { - source /opt/ic/bin/functions.sh log_start "$(basename $0)" install_guestos log_end "$(basename $0)" diff --git a/ic-os/components/setupos-scripts/install-hostos.sh b/ic-os/components/setupos-scripts/install-hostos.sh index bb3d3b2f424..4557a02957c 100755 --- a/ic-os/components/setupos-scripts/install-hostos.sh +++ b/ic-os/components/setupos-scripts/install-hostos.sh @@ -3,6 +3,8 @@ set -o nounset set -o pipefail +source /opt/ic/bin/functions.sh + SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" @@ -101,7 +103,6 @@ function resize_partition() { # Establish run order main() { - source /opt/ic/bin/functions.sh log_start "$(basename $0)" install_hostos configure_efi diff --git a/ic-os/components/setupos-scripts/setup-disk.sh b/ic-os/components/setupos-scripts/setup-disk.sh index f7546bec499..38437fcf20b 100755 --- a/ic-os/components/setupos-scripts/setup-disk.sh +++ b/ic-os/components/setupos-scripts/setup-disk.sh @@ -3,6 +3,8 @@ set -o nounset set -o pipefail +source /opt/ic/bin/functions.sh + SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" @@ -60,7 +62,6 @@ function setup_storage() { # Establish run order main() { - source /opt/ic/bin/functions.sh log_start "$(basename $0)" purge_partitions setup_storage diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 1fc04a959fc..181eb119487 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -3,9 +3,12 @@ set -o nounset set -o pipefail +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -CONFIG_DIR="/var/ic/config" +CONFIG_DIR="/config" function mount_config_partition() { echo "* Mounting hostOS config partition..." @@ -27,19 +30,27 @@ function copy_config_files() { fi echo "* Copying SSH authorized keys..." - if [ -d "${CONFIG_DIR}/ssh_authorized_keys" ]; then - cp -r ${CONFIG_DIR}/ssh_authorized_keys /media/ - log_and_halt_installation_on_error "${?}" "Unable to copy SSH authorized keys to hostOS config partition." + ssh_authorized_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') + if [ -n "${ssh_authorized_keys}" ] && [ "${ssh_authorized_keys}" != "null" ]; then + if [ -d "${ssh_authorized_keys}" ]; then + cp -a "${ssh_authorized_keys}" /media/ + log_and_halt_installation_on_error "${?}" "Unable to copy SSH authorized keys to hostOS config partition." + else + log_and_halt_installation_on_error "1" "Directory '${ssh_authorized_keys}' does not exist." + fi else - log_and_halt_installation_on_error "1" "Directory 'ssh_authorized_keys' does not exist." + echo >&2 "Warning: SSH authorized keys path is not configured." fi echo "* Copying node operator private key..." - if [ -f "${CONFIG_DIR}/node_operator_private_key.pem" ]; then - cp ${CONFIG_DIR}/node_operator_private_key.pem /media/ + node_operator_private_key_path=$(get_config_value '.icos_settings.node_operator_private_key_path') + if [ "${node_operator_private_key_path}" != "null" ] && [ -f "${node_operator_private_key_path}" ]; then + cp "${node_operator_private_key_path}" /media/ log_and_halt_installation_on_error "${?}" "Unable to copy node operator private key to hostOS config partition." + elif [ "${node_operator_private_key_path}" = "null" ]; then + echo >&2 "Warning: Node operator private key path is not configured." else - echo "node_operator_private_key.pem does not exist, requiring HSM." + echo >&2 "Warning: node_operator_private_key.pem does not exist, requiring HSM." fi echo "* Copying deployment.json to config partition..." @@ -47,8 +58,22 @@ function copy_config_files() { log_and_halt_installation_on_error "${?}" "Unable to copy deployment.json to hostOS config partition." echo "* Copying NNS public key to hostOS config partition..." - cp /data/nns_public_key.pem /media/ + nns_public_key_path=$(get_config_value '.icos_settings.nns_public_key_path') + cp "${nns_public_key_path}" /media/ log_and_halt_installation_on_error "${?}" "Unable to copy NNS public key to hostOS config partition." + + echo "* Converting 'config.json' to hostOS config file 'config-hostos.json'..." + /opt/ic/bin/config generate-hostos-config + log_and_halt_installation_on_error "${?}" "Unable to generate hostos configuration." + + # TODO: NODE-1466: Configuration revamp (HostOS and GuestOS integration) + # echo "* Copying 'config-hostos.json' to hostOS config partition..." + # if [ -f "/var/ic/config/config-hostos.json" ]; then + # cp /var/ic/config/config-hostos.json /media/config.json + # log_and_halt_installation_on_error "${?}" "Unable to copy 'config-hostos.json' to hostOS config partition." + # else + # log_and_halt_installation_on_error "1" "Configuration file 'config-hostos.json' does not exist." + # fi } function insert_hsm_if_necessary() { @@ -82,7 +107,6 @@ function unmount_config_partition() { # Establish run order main() { - source /opt/ic/bin/functions.sh log_start "$(basename $0)" mount_config_partition copy_config_files diff --git a/ic-os/components/setupos-scripts/setupos.sh b/ic-os/components/setupos-scripts/setupos.sh index 413f6453c1e..67fcbeedc77 100755 --- a/ic-os/components/setupos-scripts/setupos.sh +++ b/ic-os/components/setupos-scripts/setupos.sh @@ -3,6 +3,8 @@ set -o nounset set -o pipefail +source /opt/ic/bin/functions.sh + SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" @@ -34,7 +36,6 @@ function reboot_setupos() { # Establish run order main() { - source /opt/ic/bin/functions.sh log_start "$(basename $0)" start_setupos /opt/ic/bin/check-setupos-age.sh diff --git a/ic-os/components/setupos.bzl b/ic-os/components/setupos.bzl index ed99261edb2..81def999eeb 100644 --- a/ic-os/components/setupos.bzl +++ b/ic-os/components/setupos.bzl @@ -5,7 +5,7 @@ Enumerate every component file dependency for SetupOS component_files = { # setupos-scripts Label("setupos-scripts/check-setupos-age.sh"): "/opt/ic/bin/check-setupos-age.sh", - Label("setupos-scripts/config.sh"): "/opt/ic/bin/config.sh", + Label("setupos-scripts/check-config.sh"): "/opt/ic/bin/check-config.sh", Label("setupos-scripts/setup-hostos-config.sh"): "/opt/ic/bin/setup-hostos-config.sh", Label("setupos-scripts/setup-disk.sh"): "/opt/ic/bin/setup-disk.sh", Label("setupos-scripts/functions.sh"): "/opt/ic/bin/functions.sh", @@ -26,9 +26,9 @@ component_files = { # misc Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", + Label("misc/config.sh"): "/opt/ic/bin/config.sh", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", Label("misc/chrony/chrony-var.service"): "/etc/systemd/system/chrony-var.service", - Label("misc/fetch-property.sh"): "/opt/ic/bin/fetch-property.sh", Label("misc/serial-getty@/setupos/serial-getty@.service"): "/etc/systemd/system/serial-getty@.service", Label("monitoring/journald.conf"): "/etc/systemd/journald.conf", diff --git a/ic-os/setupos/defs.bzl b/ic-os/setupos/defs.bzl index 325ebb995fa..6483645fd41 100644 --- a/ic-os/setupos/defs.bzl +++ b/ic-os/setupos/defs.bzl @@ -31,6 +31,7 @@ def image_deps(mode, _malicious = False): "bootfs": {}, "rootfs": { "//rs/ic_os/release:setupos_tool": "/opt/ic/bin/setupos_tool:0755", + "//rs/ic_os/release:config": "/opt/ic/bin/config:0755", }, # Set various configuration values diff --git a/rs/ic_os/network/src/info.rs b/rs/ic_os/network/src/info.rs deleted file mode 100644 index 88109c8e71a..00000000000 --- a/rs/ic_os/network/src/info.rs +++ /dev/null @@ -1,154 +0,0 @@ -use std::net::Ipv6Addr; - -use anyhow::{bail, Context, Result}; - -use config::config_ini::ConfigMap; - -#[derive(Debug)] -pub struct NetworkInfo { - // Config files can specify ipv6 prefix, address and prefix, or just address. - // ipv6_address takes precedence. Some tests provide only the address. - // Should be kept as a string until parsing later. - pub ipv6_prefix: Option, - pub ipv6_address: Option, - pub ipv6_subnet: u8, - pub ipv6_gateway: Ipv6Addr, -} - -fn is_valid_prefix(ipv6_prefix: &str) -> bool { - ipv6_prefix.len() <= 19 && format!("{ipv6_prefix}::").parse::().is_ok() -} - -impl NetworkInfo { - pub fn from_config_map(config_map: &ConfigMap) -> Result { - // Per PFOPS - this will never not be 64 - let ipv6_subnet = 64_u8; - - let ipv6_prefix = match config_map.get("ipv6_prefix") { - Some(ipv6_prefix) => { - // Prefix should have a max length of 19 ("1234:6789:1234:6789") - // It could have fewer characters though. Parsing as an ip address with trailing '::' should work. - if !is_valid_prefix(ipv6_prefix) { - bail!("Invalid ipv6 prefix: {}", ipv6_prefix); - } - Some(ipv6_prefix.clone()) - } - None => None, - }; - - // Optional ipv6_address - for testing. Takes precedence over ipv6_prefix. - let ipv6_address = match config_map.get("ipv6_address") { - Some(address) => { - // ipv6_address might be formatted with the trailing suffix. Remove it. - let ipv6_subnet = format!("/{}", ipv6_subnet); - let address = address.strip_suffix(&ipv6_subnet).unwrap_or(address); - let address = address - .parse::() - .context(format!("Invalid ipv6 address: {}", address))?; - Some(address) - } - None => None, - }; - - if ipv6_address.is_none() && ipv6_prefix.is_none() { - bail!("Missing config parameter: need at least one of ipv6_prefix or ipv6_address"); - } - - let ipv6_gateway = config_map - .get("ipv6_gateway") - .context("Missing config parameter: ipv6_gateway")?; - let ipv6_gateway = ipv6_gateway - .parse::() - .context(format!("Invalid ipv6 address: {}", ipv6_gateway))?; - - Ok(NetworkInfo { - ipv6_prefix, - ipv6_subnet, - ipv6_gateway, - ipv6_address, - }) - } -} - -#[cfg(test)] -pub mod tests { - use std::collections::HashMap; - - use super::*; - #[test] - fn test_is_valid_prefix() { - assert!(is_valid_prefix("2a00:1111:1111:1111")); - assert!(is_valid_prefix("2a00:111:11:11")); - assert!(is_valid_prefix("2602:fb2b:100:10")); - } - - #[test] - fn test_from_config_map() { - // Example config.ini - let config_map = HashMap::from([ - ("ipv6_prefix".to_string(), "2a00:fb01:400:100".to_string()), - ( - "ipv6_gateway".to_string(), - "2a00:fb01:400:100::1".to_string(), - ), - ]); - assert!(NetworkInfo::from_config_map(&config_map).is_ok()); - - // With ipv6_address and ipv6_prefix - let config_map = HashMap::from([ - ("ipv6_prefix".to_string(), "2a00:fb01:400:100".to_string()), - ( - "ipv6_gateway".to_string(), - "2a00:fb01:400:100::1".to_string(), - ), - ( - "ipv6_address".to_string(), - "2a00:fb01:400:100::3".to_string(), - ), - ]); - assert!(NetworkInfo::from_config_map(&config_map).is_ok()); - - // No subnet - let config_map = HashMap::from([ - ("ipv6_prefix".to_string(), "2a00:fb01:400:100".to_string()), - ( - "ipv6_gateway".to_string(), - "2a00:fb01:400:100::1".to_string(), - ), - ]); - assert!(NetworkInfo::from_config_map(&config_map).is_ok()); - - // Need address or prefix - let config_map = HashMap::from([ - ( - "ipv6_address".to_string(), - "2a00:fb01:400:100::1".to_string(), - ), - ( - "ipv6_gateway".to_string(), - "2a00:fb01:400:100::1".to_string(), - ), - ]); - assert!(NetworkInfo::from_config_map(&config_map).is_ok()); - - // Need prefix or address, gateway - let config_map = HashMap::from([( - "ipv6_gateway".to_string(), - "2a00:fb01:400:100::1".to_string(), - )]); - assert!(NetworkInfo::from_config_map(&config_map).is_err()); - let config_map = - HashMap::from([("ipv6_prefix".to_string(), "2a00:fb01:400:100".to_string())]); - assert!(NetworkInfo::from_config_map(&config_map).is_err()); - - // With ipv6_address with subnet len - let config_map = HashMap::from([ - ( - "ipv6_gateway".to_string(), - "2a00:fb01:400:100::1".to_string(), - ), - ("ipv6_address".to_string(), "fd00:2:1:1::11/64".to_string()), - ]); - assert!(NetworkInfo::from_config_map(&config_map).is_ok()); - } -} diff --git a/rs/ic_os/network/src/interfaces.rs b/rs/ic_os/network/src/interfaces.rs index 2b7895be1bf..3f041e3856f 100644 --- a/rs/ic_os/network/src/interfaces.rs +++ b/rs/ic_os/network/src/interfaces.rs @@ -22,11 +22,11 @@ pub struct Interface { pub fn has_ipv6_connectivity( interface: &Interface, generated_ipv6: &Ipv6Addr, - ipv6_subnet: u8, + ipv6_prefix_length: u8, ping_target: &str, ) -> Result { // Format with the prefix length - let ip = format!("{}/{}", generated_ipv6, ipv6_subnet); + let ip = format!("{}/{}", generated_ipv6, ipv6_prefix_length); let interface_down_func = || { eprintln!("Removing ip address and bringing interface down"); get_command_stdout("ip", ["addr", "del", &ip, "dev", &interface.name])?; diff --git a/rs/ic_os/network/src/lib.rs b/rs/ic_os/network/src/lib.rs index 2e8f130fdd5..caa65ca1642 100644 --- a/rs/ic_os/network/src/lib.rs +++ b/rs/ic_os/network/src/lib.rs @@ -5,11 +5,10 @@ use anyhow::{Context, Result}; use crate::mac_address::generate_mac_address; use crate::node_type::NodeType; use crate::systemd::generate_systemd_config_files; -use info::NetworkInfo; +use config::types::NetworkSettings; use ipv6::generate_ipv6_address; use mac_address::FormattedMacAddress; -pub mod info; pub mod interfaces; pub mod ipv6; pub mod mac_address; @@ -19,25 +18,25 @@ pub mod systemd; /// Write SetupOS or HostOS systemd network configuration. /// Requires superuser permissions to run `ipmitool` and write to the systemd directory pub fn generate_network_config( - network_info: &NetworkInfo, - mgmt_mac: Option<&str>, - deployment_name: Option<&str>, + network_settings: &NetworkSettings, + deployment_name: &str, node_type: NodeType, output_directory: &Path, ) -> Result<()> { - if let Some(address) = network_info.ipv6_address { + if let Some(address) = network_settings.ipv6_address { eprintln!("Found ipv6 address in config"); - return generate_systemd_config_files(output_directory, network_info, None, &address); + return generate_systemd_config_files(output_directory, network_settings, None, &address); }; - let deployment_name = deployment_name - .context("Error: Deployment name not found when attempting to generate mac address")?; - - let mac = generate_mac_address(deployment_name, &node_type, mgmt_mac)?; + let mac = generate_mac_address( + deployment_name, + &node_type, + network_settings.mgmt_mac.as_deref(), + )?; eprintln!("Using generated mac (unformatted) {}", mac.get()); eprintln!("Generating ipv6 address"); - let ipv6_prefix = network_info + let ipv6_prefix = network_settings .ipv6_prefix .clone() .context("ipv6_prefix required in config to generate ipv6 address")?; @@ -47,7 +46,7 @@ pub fn generate_network_config( let formatted_mac = FormattedMacAddress::from(&mac); generate_systemd_config_files( output_directory, - network_info, + network_settings, Some(&formatted_mac), &ipv6_address, ) diff --git a/rs/ic_os/network/src/systemd.rs b/rs/ic_os/network/src/systemd.rs index d2e1c2685c6..bd58f0f2916 100644 --- a/rs/ic_os/network/src/systemd.rs +++ b/rs/ic_os/network/src/systemd.rs @@ -5,9 +5,9 @@ use std::process::Command; use anyhow::{Context, Result}; -use crate::info::NetworkInfo; use crate::interfaces::{get_interfaces, has_ipv6_connectivity, Interface}; use crate::mac_address::FormattedMacAddress; +use config::types::NetworkSettings; pub static DEFAULT_SYSTEMD_NETWORK_DIR: &str = "/run/systemd/network"; @@ -149,7 +149,7 @@ fn generate_and_write_systemd_files( pub fn generate_systemd_config_files( output_directory: &Path, - network_info: &NetworkInfo, + network_settings: &NetworkSettings, generated_mac: Option<&FormattedMacAddress>, ipv6_address: &Ipv6Addr, ) -> Result<()> { @@ -157,13 +157,18 @@ pub fn generate_systemd_config_files( interfaces.sort_by(|a, b| a.speed_mbps.cmp(&b.speed_mbps)); eprintln!("Interfaces sorted by speed: {:?}", interfaces); - let ping_target = network_info.ipv6_gateway.to_string(); + let ping_target = network_settings.ipv6_gateway.to_string(); // old nodes are still configured with a local IPv4 interface connection // local IPv4 interfaces must be filtered out let ipv6_interfaces: Vec<&Interface> = interfaces .iter() .filter(|i| { - match has_ipv6_connectivity(i, ipv6_address, network_info.ipv6_subnet, &ping_target) { + match has_ipv6_connectivity( + i, + ipv6_address, + network_settings.ipv6_prefix_length, + &ping_target, + ) { Ok(result) => result, Err(e) => { eprintln!("Error testing connectivity on {}: {}", &i.name, e); @@ -183,13 +188,17 @@ pub fn generate_systemd_config_files( eprintln!("Using fastest interface: {:?}", fastest_interface); // Format the ip address to include the subnet length. See `man systemd.network`. - let ipv6_address = format!("{}/{}", &ipv6_address.to_string(), network_info.ipv6_subnet); + let ipv6_address = format!( + "{}/{}", + &ipv6_address.to_string(), + network_settings.ipv6_prefix_length + ); generate_and_write_systemd_files( output_directory, fastest_interface, generated_mac, &ipv6_address, - &network_info.ipv6_gateway.to_string(), + &network_settings.ipv6_gateway.to_string(), )?; print!("Restarting systemd networkd"); diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index c05a053ca42..20442549009 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -3,11 +3,11 @@ use std::path::Path; use anyhow::{anyhow, Context, Result}; use clap::{Parser, Subcommand}; -use config::config_ini::config_map_from_path; +use config::config_ini::get_config_ini_settings; use config::deployment_json::get_deployment_settings; +use config::types::NetworkSettings; use config::{DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH, DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH}; use network::generate_network_config; -use network::info::NetworkInfo; use network::ipv6::generate_ipv6_address; use network::mac_address::{generate_mac_address, FormattedMacAddress}; use network::node_type::NodeType; @@ -56,78 +56,102 @@ pub fn main() -> Result<()> { match opts.command { Some(Commands::GenerateNetworkConfig { output_directory }) => { - let config_map = config_map_from_path(Path::new(&opts.config)) - .context("Please specify a valid config file with '--config'")?; - eprintln!("Using config: {:?}", config_map); - - let network_info = NetworkInfo::from_config_map(&config_map)?; - eprintln!("Network info config: {:?}", &network_info); - - let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)); - - let deployment_name: Option<&str> = match &deployment_settings { - Ok(deployment) => Some(deployment.deployment.name.as_str()), - Err(e) => { - eprintln!("Error retrieving deployment file: {e}. Continuing without it"); - None - } - }; - - let mgmt_mac: Option<&str> = match &deployment_settings { - Ok(deployment) => deployment.deployment.mgmt_mac.as_deref(), - Err(_) => None, + let config_ini_settings = get_config_ini_settings(Path::new(&opts.config))?; + + let deployment_json_settings = + get_deployment_settings(Path::new(&opts.deployment_file))?; + eprintln!("Deployment config: {:?}", deployment_json_settings); + + // TODO: NODE-1466: Remove in configuration revamp (HostOS and GuestOS integration). + // Once HostOS is using the config struct, all config will be contained there + // and we won't need to read config.ini and deployment.json directly. + let network_settings = NetworkSettings { + ipv6_prefix: config_ini_settings.ipv6_prefix, + ipv6_address: config_ini_settings.ipv6_address, + ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, + ipv6_gateway: config_ini_settings.ipv6_gateway, + ipv4_address: config_ini_settings.ipv4_address, + ipv4_gateway: config_ini_settings.ipv4_gateway, + ipv4_prefix_length: config_ini_settings.ipv4_prefix_length, + domain: config_ini_settings.domain, + mgmt_mac: deployment_json_settings.deployment.mgmt_mac, }; + eprintln!("Network settings config: {:?}", &network_settings); generate_network_config( - &network_info, - mgmt_mac, - deployment_name, + &network_settings, + deployment_json_settings.deployment.name.as_str(), NodeType::HostOS, Path::new(&output_directory), ) } Some(Commands::GenerateIpv6Address { node_type }) => { - let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)) - .context("Please specify a valid deployment file with '--deployment-file'")?; - eprintln!("Deployment config: {:?}", deployment_settings); - - let config_map = config_map_from_path(Path::new(&opts.config)) - .context("Please specify a valid config file with '--config'")?; - eprintln!("Using config: {:?}", config_map); - - let network_info = NetworkInfo::from_config_map(&config_map)?; - eprintln!("Network info config: {:?}", &network_info); + let config_ini_settings = get_config_ini_settings(Path::new(&opts.config))?; + + let deployment_json_settings = + get_deployment_settings(Path::new(&opts.deployment_file))?; + eprintln!("Deployment config: {:?}", deployment_json_settings); + + // TODO: NODE-1466: Remove in configuration revamp (HostOS and GuestOS integration). + // Once HostOS is using the config struct, all config will be contained there + // and we won't need to read config.ini and deployment.json directly. + let network_settings = NetworkSettings { + ipv6_prefix: config_ini_settings.ipv6_prefix, + ipv6_address: config_ini_settings.ipv6_address, + ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, + ipv6_gateway: config_ini_settings.ipv6_gateway, + ipv4_address: config_ini_settings.ipv4_address, + ipv4_gateway: config_ini_settings.ipv4_gateway, + ipv4_prefix_length: config_ini_settings.ipv4_prefix_length, + domain: config_ini_settings.domain, + mgmt_mac: deployment_json_settings.deployment.mgmt_mac, + }; + eprintln!("Network settings config: {:?}", &network_settings); let node_type = node_type.parse::()?; let mac = generate_mac_address( - &deployment_settings.deployment.name, + &deployment_json_settings.deployment.name, &node_type, - deployment_settings.deployment.mgmt_mac.as_deref(), + network_settings.mgmt_mac.as_deref(), )?; - let ipv6_prefix = network_info + let ipv6_prefix = network_settings .ipv6_prefix .context("ipv6_prefix required in config to generate ipv6 address")?; let ipv6_address = generate_ipv6_address(&ipv6_prefix, &mac)?; - println!("{}", to_cidr(ipv6_address, network_info.ipv6_subnet)); + println!( + "{}", + to_cidr(ipv6_address, network_settings.ipv6_prefix_length) + ); Ok(()) } Some(Commands::GenerateMacAddress { node_type }) => { - let config_map = config_map_from_path(Path::new(&opts.config)) - .context("Please specify a valid config file with '--config'")?; - eprintln!("Using config: {:?}", config_map); - - let network_info = NetworkInfo::from_config_map(&config_map)?; - eprintln!("Network info config: {:?}", &network_info); - - let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)) - .context("Please specify a valid deployment file with '--deployment-file'")?; - eprintln!("Deployment config: {:?}", deployment_settings); + let config_ini_settings = get_config_ini_settings(Path::new(&opts.config))?; + + let deployment_json_settings = + get_deployment_settings(Path::new(&opts.deployment_file))?; + eprintln!("Deployment config: {:?}", deployment_json_settings); + + // TODO: NODE-1466: Remove in configuration revamp (HostOS and GuestOS integration). + // Once HostOS is using the config struct, all config will be contained there + // and we won't need to read config.ini and deployment.json directly. + let network_settings = NetworkSettings { + ipv6_prefix: config_ini_settings.ipv6_prefix, + ipv6_address: config_ini_settings.ipv6_address, + ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, + ipv6_gateway: config_ini_settings.ipv6_gateway, + ipv4_address: config_ini_settings.ipv4_address, + ipv4_gateway: config_ini_settings.ipv4_gateway, + ipv4_prefix_length: config_ini_settings.ipv4_prefix_length, + domain: config_ini_settings.domain, + mgmt_mac: deployment_json_settings.deployment.mgmt_mac, + }; + eprintln!("Network settings config: {:?}", &network_settings); let node_type = node_type.parse::()?; let mac = generate_mac_address( - &deployment_settings.deployment.name, + &deployment_json_settings.deployment.name, &node_type, - deployment_settings.deployment.mgmt_mac.as_deref(), + network_settings.mgmt_mac.as_deref(), )?; let mac = FormattedMacAddress::from(&mac); println!("{}", mac.get()); diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 0bcda31d0ba..bc88fb501a0 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -1,13 +1,14 @@ use std::path::Path; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; -use config::config_ini::config_map_from_path; -use config::deployment_json::get_deployment_settings; -use config::{DEFAULT_SETUPOS_CONFIG_INI_FILE_PATH, DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH}; +use config::types::SetupOSConfig; +use config::{ + deserialize_config, DEFAULT_SETUPOS_CONFIG_INI_FILE_PATH, DEFAULT_SETUPOS_CONFIG_OBJECT_PATH, + DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH, +}; use network::generate_network_config; -use network::info::NetworkInfo; use network::ipv6::generate_ipv6_address; use network::mac_address::{generate_mac_address, FormattedMacAddress}; use network::node_type::NodeType; @@ -55,79 +56,63 @@ pub fn main() -> Result<()> { match opts.command { Some(Commands::GenerateNetworkConfig { output_directory }) => { - let config_map = config_map_from_path(Path::new(&opts.config)) - .context("Please specify a valid config file with '--config'")?; - eprintln!("Using config: {:?}", config_map); - - let network_info = NetworkInfo::from_config_map(&config_map)?; - eprintln!("Network info config: {:?}", &network_info); - - let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)); - let deployment_name: Option<&str> = match &deployment_settings { - Ok(deployment) => Some(deployment.deployment.name.as_str()), - Err(e) => { - eprintln!("Error retrieving deployment file: {e}. Continuing without it"); - None - } - }; - - let mgmt_mac: Option<&str> = match &deployment_settings { - Ok(deployment) => deployment.deployment.mgmt_mac.as_deref(), - Err(_) => None, - }; + let setup_config: SetupOSConfig = + deserialize_config(DEFAULT_SETUPOS_CONFIG_OBJECT_PATH)?; + + eprintln!( + "Network settings config: {:?}", + &setup_config.network_settings + ); generate_network_config( - &network_info, - mgmt_mac, - deployment_name, + &setup_config.network_settings, + &setup_config.icos_settings.hostname, NodeType::SetupOS, Path::new(&output_directory), ) } Some(Commands::GenerateIpv6Address { node_type }) => { - let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)) - .context("Please specify a valid deployment file with '--deployment-file'")?; - eprintln!("Deployment config: {:?}", deployment_settings); - - let config_map = config_map_from_path(Path::new(&opts.config)) - .context("Please specify a valid config file with '--config'")?; - eprintln!("Using config: {:?}", config_map); - - let network_info = NetworkInfo::from_config_map(&config_map)?; - eprintln!("Network info config: {:?}", &network_info); + let setup_config: SetupOSConfig = + deserialize_config(DEFAULT_SETUPOS_CONFIG_OBJECT_PATH)?; let node_type = node_type.parse::()?; let mac = generate_mac_address( - &deployment_settings.deployment.name, + &setup_config.icos_settings.hostname, &node_type, - deployment_settings.deployment.mgmt_mac.as_deref(), + setup_config.network_settings.mgmt_mac.as_deref(), )?; - let ipv6_prefix = network_info - .ipv6_prefix - .context("ipv6_prefix required in config to generate ipv6 address")?; + let ipv6_prefix = setup_config.network_settings.ipv6_prefix.ok_or_else(|| { + anyhow!("ipv6_prefix required in config to generate ipv6 address") + })?; let ipv6_address = generate_ipv6_address(&ipv6_prefix, &mac)?; - println!("{}", to_cidr(ipv6_address, network_info.ipv6_subnet)); + println!( + "{}", + to_cidr( + ipv6_address, + setup_config.network_settings.ipv6_prefix_length + ) + ); + Ok(()) } Some(Commands::GenerateMacAddress { node_type }) => { - let config_map = config_map_from_path(Path::new(&opts.config)) - .context("Please specify a valid config file with '--config'")?; - eprintln!("Using config: {:?}", config_map); + let setup_config: SetupOSConfig = + deserialize_config(DEFAULT_SETUPOS_CONFIG_OBJECT_PATH)?; - let network_info = NetworkInfo::from_config_map(&config_map)?; - eprintln!("Network info config: {:?}", &network_info); - - let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)) - .context("Please specify a valid deployment file with '--deployment-file'")?; - eprintln!("Deployment config: {:?}", deployment_settings); + eprintln!( + "Network settings config: {:?}", + &setup_config.network_settings + ); let node_type = node_type.parse::()?; + let mac = generate_mac_address( - &deployment_settings.deployment.name, + &setup_config.icos_settings.hostname, &node_type, - deployment_settings.deployment.mgmt_mac.as_deref(), + setup_config.network_settings.mgmt_mac.as_deref(), )?; + let mac = FormattedMacAddress::from(&mac); println!("{}", mac.get()); Ok(()) From b56317904e8f212e20a9a47a469d63e752ee8c83 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 18 Sep 2024 21:22:45 +0000 Subject: [PATCH 002/241] Partially update configuration documentation --- .../guestos/bootstrap-ic-node.sh | 2 - ic-os/docs/Configuration.adoc | 65 ++++++++++++++----- ic-os/docs/README.adoc | 1 - ic-os/guestos/docs/Boot.adoc | 6 +- ic-os/guestos/docs/ConfigStore.adoc | 45 ------------- ic-os/guestos/docs/DiskLayout.adoc | 2 - ic-os/guestos/docs/README.adoc | 1 - 7 files changed, 53 insertions(+), 69 deletions(-) delete mode 100644 ic-os/guestos/docs/ConfigStore.adoc diff --git a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh index 6b1f5f8b134..08bfc78ba45 100755 --- a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh @@ -3,8 +3,6 @@ # Provision a node based on an injected "ic-bootstrap.tar" file. This script # is meant to be run as a prerequisite before launching orchestrator/replica. # -# The configuration format is described in guestos/docs/ConfigStore.adoc -# # The tar file can be supplied using one of two methods: # - as "ic-bootstrap.tar" stored on a (virtual) removable media attached # on first boot diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index d505d0359b3..8d42ce11442 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -2,27 +2,25 @@ Each IC-OS has a 100 MB config partition. All IC-OS config partitions are initialized to be empty, except for SetupOS. -In production, configuration is propagated from a partition on the USB installer through each of SetupOS, HostOS and GuestOS: +In production, configuration is propagated from a partition on the USB installer through each of SetupOS, HostOS and GuestOS. +This process is controlled by the (link:../../rs/ic_os/config/README.md[ic-os config tool]) and an assortment of bash scripts. -* SetupOS reads and validates its configuration files from `/config/` -* SetupOS copies sanitized configuration files from `/config/` to `/var/ic/config/` -* SetupOS copies its configuration files from `/var/ic/config/` to the HostOS config partition. -* HostOS reads the configuration files from `/boot/config`. These files are used to populate the GuestOS config partition through a more complicated process described below. +All access to the config partition should be done through the ic-os config tool. -== Detailed configuration steps +== User-facing configuration files -=== SetupOS -> HostOS - -SetupOS validates, sanitizes, and copies all of its configuration files to the HostOS config partition: +SetupOS constructs its config struct from the following user-facing configuration files: config.ini # Data center-specific network settings ssh_authorized_keys # SSH private keys node_operator_private_key.pem # Node Operator private key created in the Node Provider onboarding deployment.json # Deployment-specific configurations nns_public_key.pem # NNS public key -Refer to link:../../rs/ic_os/config/README.md[rs/ic_os/config] & link:../components/setupos-scripts/setup-hostos-config.sh[setup-hostos-config.sh] +Refer to link:../../rs/ic_os/config/README.md[rs/ic_os/config] and link:../components/setupos-scripts/setup-hostos-config.sh[setup-hostos-config.sh] for more details. + +== HostOS -> GuestOS -=== HostOS -> GuestOS +TODO: update... HostOS builds the "bootstrap config image". Refer to link:../components/hostos-scripts/build-bootstrap-config-image.sh[build-bootstrap-config-image.sh] @@ -44,14 +42,32 @@ GuestOS only reads a predefined set of files from the bootstrap config image (i. [NOTE] The reason for the bootstrap config image redirection is to ensure that GuestOS maintains control over what goes onto its config partition. Theoretically, a malicious Node Technician could modify their HostOS image and place any file onto the bootstrap config image. However, GuestOS will only copy a predefined set of files to its config partition. -== GuestOS configuration files +== GuestOS config partition + +TODO: update... + +The config partition stores information that must be preserved across system upgrades and needs to be available during early boot time. Consequently, this information cannot reside within the encrypted payload data partition. + +Currently, all contents in the config partition are stored as plain-text without integrity protection. + +These files are stored in `/boot/config` or `/var/lib/ic`. To see where each configuration file is stored, refer to link:../../components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh[bootstrap-ic-node] -To learn more about the GuestOS configuration files, link:../guestos/docs/ConfigStore.adoc[see the GuestOS ConfigStore.adoc] +=== CONFIGURED file + +This file serves as a tag to indicate that the one-time bootstrap configuration has been completed. If the `/boot/config/CONFIGURED` file is not present, the boot sequence will search for a virtual USB stick (the bootstrap config image) containing the injected configuration files, and create the file. + +=== store.keyfile + +This file contains the key material used to derive the wrapping key for all block device encryption. The `store.keyfile` is created during the first boot, and encrypted partitions are configured with it. + +In the absence of a sealing key (which will be available in SEV-protected trusted execution environments), the `store.keyfile` is stored as plain-text. Once a sealing key becomes available, it should be used to wrap the contents of this file. == Implementation notes === Guidance for adding configuration bits +TODO: update... + To add a new configuration file/directory: 1. Add handling to `build-bootstrap-config-image.sh` to include the new file/directory in the bootstrap config image. @@ -72,10 +88,10 @@ Consider that values may be controlled by an attacker on boot. Bootstrapping a n *Interpretation of configuration bits*: Any script or service in the system may pull configuration bits out of /boot/config to customize its behavior. E.g. if adding parameter-driven customization of ic.json5, then augment the generate-replica-config.sh script to pull the configuration values and substitute them into the generated configuration. -*Documentation*: Please keep documentation up-to-date (link:ConfigStore-SetupOSHostOS.adoc[SetupOS/HostOS config store], link:../guestos/docs/ConfigStore.adoc[GuestOS config store]) - === Testing +TODO: update... + * *bootstrap-ic-node.sh* can be temporarily tweaked (internally adapt paths, then run the process_bootstrap function): ** run stand-alone ** verify that the config image is unpacked @@ -86,3 +102,22 @@ Consider that values may be controlled by an attacker on boot. Bootstrapping a n * *generate-replica-config.sh* can be run stand-alone to verify that it produces the intended ic.json5 configuration from the template. After all is done, it is advised to prepare a configuration for a single node and boot it in a VM before conducting testnet deployments. + +=== Injecting external state + +TODO: update... + +*Typical bootstrap process:* On first boot, the system will perform technical initialization (filesystems, etc.) and afterwards, initialize itself to act as a node in the IC. The node is initialized using key generation on the node itself (such that the private key never leaves the node) and through joining the IC (the node gets the rest of its state via joining the IC). "Registration" to the target IC is initiated by the node itself by sending a Node Operator-signed "join" request to its NNS. + +However, the typical bootstrap process can be modified such that the node is initialized using externally generated private keys and an externally generated initial state. All "registration" to the target IC is assumed to have been performed by other means. + +The behavior is triggered through the presence of the following files: + +- ic_crypto +- ic_registry_local_store + +This behavior is suitable for the following use cases: + +- Bootstrapping an IC instance: In this case, suitable state for all nodes is generated by ic-prep and then distributed across multiple nodes. This is used, for example, during testnet setup. + +- Externally controlled join of a node to a subnet: In this case, ic-prep is used to prepare key material to the node, while ic-admin is used to modify the target NNS such that it "accepts" the new node as part of the IC. diff --git a/ic-os/docs/README.adoc b/ic-os/docs/README.adoc index 966475c5c75..09ef5d3c9a8 100644 --- a/ic-os/docs/README.adoc +++ b/ic-os/docs/README.adoc @@ -7,4 +7,3 @@ Refer to detailed documentation on: * link:Components{outfilesuffix}[Components] * link:SELinux{outfilesuffix}[SELinux security policy] * link:Configuration{outfilesuffix}[Configuration] -* link:ConfigStore-SetupOSHostOS{outfilesuffix}[SetupOS and HostOS config store] diff --git a/ic-os/guestos/docs/Boot.adoc b/ic-os/guestos/docs/Boot.adoc index a744509ec75..aa4e4803926 100644 --- a/ic-os/guestos/docs/Boot.adoc +++ b/ic-os/guestos/docs/Boot.adoc @@ -57,7 +57,7 @@ not held in +/etc/fstab+ but is generated by the shell script Afterwards, the first three partitions are mounted as +/boot/efi+, +/boot/grub+ and +/boot/config+, respectively. The +config+ partition is used as (small) store for data that is preserved across upgrades -and is available at early boot time already (see link:ConfigStore{outfilesuffix}[config store]). +and is available at early boot time already. == Save machine-id @@ -167,8 +167,8 @@ depends on mount of all filesystems. This is only executed once on first boot after provisioning. It looks for a "virtual USB stick" attached to the VM that contains a tar file with initial configuration -for parts of the system (see link:ConfigStore{outfilesuffix}[config store] for a description). Required -files in the +config+ partition as well as payload store are created. +for parts of the system. Required files in the +config+ partition as well as +payload store are created. == Deploy updated ssh account keys diff --git a/ic-os/guestos/docs/ConfigStore.adoc b/ic-os/guestos/docs/ConfigStore.adoc deleted file mode 100644 index f0d0c8e382d..00000000000 --- a/ic-os/guestos/docs/ConfigStore.adoc +++ /dev/null @@ -1,45 +0,0 @@ -= GuestOS Config Store - -This document calls out some of the contents of the GuestOS *config* partition (*/dev/vda3* in the GuestOS disk image). The config partition stores information that must be preserved across system upgrades and needs to be available during early boot time. Consequently, this information cannot reside within the encrypted payload data partition. - -Currently, all contents in the config partition are stored as plain-text without integrity protection. - -These files are stored in `/boot/config` or `/var/lib/ic`. To see where each configuration file is stored, refer to link:../../components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh[bootstrap-ic-node] - -== Production configuration files - -Not all configuration files and directories are required for GuestOS to run in production, as certain configuration files exist solely for testing and development purposes. - -The following files and directories *are* required for GuestOS to run in production. - -=== CONFIGURED - -This file serves as a tag to indicate that the one-time bootstrap configuration has been completed. If the `/boot/config/CONFIGURED` file is not present, the boot sequence will search for a virtual USB stick (the bootstrap config image) containing the injected configuration files, and create the file. - -=== store.keyfile - -This file contains the key material used to derive the wrapping key for all block device encryption. The `store.keyfile` is created during the first boot, and encrypted partitions are configured with it. - -In the absence of a sealing key (which will be available in SEV-protected trusted execution environments), the `store.keyfile` is stored as plain-text. Once a sealing key becomes available, it should be used to wrap the contents of this file. - -== Development configuration files - -These configuration files should only be used for development and testing purposes. - -== Injecting external state - -*Typical bootstrap process:* On first boot, the system will perform technical initialization (filesystems, etc.) and afterwards, initialize itself to act as a node in the IC. The node is initialized using key generation on the node itself (such that the private key never leaves the node) and through joining the IC (the node gets the rest of its state via joining the IC). "Registration" to the target IC is initiated by the node itself by sending a Node Operator-signed "join" request to its NNS. - -However, the typical bootstrap process can be modified such that the node is initialized using externally generated private keys and an externally generated initial state. All "registration" to the target IC is assumed to have been performed by other means. - -The behavior is triggered through the presence of the following files: - -- ic_crypto -- ic_registry_local_store - -This behavior is suitable for the following use cases: - -- Bootstrapping an IC instance: In this case, suitable state for all nodes is generated by ic-prep and then distributed across multiple nodes. This is used, for example, during testnet setup. - -- Externally controlled join of a node to a subnet: In this case, ic-prep is used to prepare key material to the node, while ic-admin is used to modify the target NNS such that it "accepts" the new node as part of the IC. - diff --git a/ic-os/guestos/docs/DiskLayout.adoc b/ic-os/guestos/docs/DiskLayout.adoc index 8a690680b9d..13dbcbd99f5 100644 --- a/ic-os/guestos/docs/DiskLayout.adoc +++ b/ic-os/guestos/docs/DiskLayout.adoc @@ -41,8 +41,6 @@ tampering). == *config* System config store Contains the config store persisted across system upgrades. -See link:ConfigStore{outfilesuffix}[config store] for a -specification of its contents. == *A_boot* / *B_boot* Boot partition for system A/B diff --git a/ic-os/guestos/docs/README.adoc b/ic-os/guestos/docs/README.adoc index 4caa2719d2c..92e95a07169 100644 --- a/ic-os/guestos/docs/README.adoc +++ b/ic-os/guestos/docs/README.adoc @@ -3,7 +3,6 @@ Refer to detailed documentation on: * link:DiskLayout{outfilesuffix}[Disk layout] -* link:ConfigStore{outfilesuffix}[GuestOS config store] * link:Boot{outfilesuffix}[Boot sequence] * link:SELinux{outfilesuffix}[SELinux security policy] * link:Interface{outfilesuffix}[GuestOS input/output interface] \ No newline at end of file From a268059dbeb3e9640e419832ca2765dd6a13a64d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 24 Sep 2024 22:13:30 +0000 Subject: [PATCH 003/241] Create ICOSDevSettings and move mgmt_mac to struct --- rs/ic_os/config/src/lib.rs | 9 +++++---- rs/ic_os/config/src/main.rs | 16 ++++++++++------ rs/ic_os/config/src/types.rs | 11 ++++++++--- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index c49f689cb53..a1bc0051f08 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -47,8 +47,8 @@ mod tests { use super::*; use std::path::PathBuf; use types::{ - GuestOSConfig, GuestOSSettings, GuestosDevConfig, HostOSConfig, HostOSSettings, - ICOSSettings, Logging, NetworkSettings, SetupOSConfig, SetupOSSettings, + GuestOSConfig, GuestOSSettings, GuestosDevSettings, HostOSConfig, HostOSSettings, + ICOSDevSettings, ICOSSettings, Logging, NetworkSettings, SetupOSConfig, SetupOSSettings, }; #[test] @@ -62,7 +62,6 @@ mod tests { ipv4_gateway: None, ipv4_prefix_length: None, domain: None, - mgmt_mac: None, }; let logging = Logging { elasticsearch_hosts: [ @@ -74,6 +73,7 @@ mod tests { .join(" "), elasticsearch_tags: None, }; + let icos_dev_settings = ICOSDevSettings { mgmt_mac: None }; let icos_settings = ICOSSettings { logging, nns_public_key_path: PathBuf::from("/path/to/key"), @@ -81,6 +81,7 @@ mod tests { hostname: "mainnet".to_string(), node_operator_private_key_path: None, ssh_authorized_keys_path: None, + icos_dev_settings, }; let setupos_settings = SetupOSSettings; let hostos_settings = HostOSSettings { @@ -92,7 +93,7 @@ mod tests { ic_crypto_path: None, ic_state_path: None, ic_registry_local_store_path: None, - guestos_dev: GuestosDevConfig::default(), + guestos_dev_settings: GuestosDevSettings::default(), }; let setupos_config_struct = SetupOSConfig { diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 65e825ee95c..10cf5b8c95d 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -7,8 +7,8 @@ use std::fs::File; use std::path::{Path, PathBuf}; use config::types::{ - GuestOSSettings, HostOSConfig, HostOSSettings, ICOSSettings, Logging, NetworkSettings, - SetupOSConfig, SetupOSSettings, + GuestOSSettings, HostOSConfig, HostOSSettings, ICOSDevSettings, ICOSSettings, Logging, + NetworkSettings, SetupOSConfig, SetupOSSettings, }; #[derive(Subcommand)] @@ -75,9 +75,6 @@ pub fn main() -> Result<()> { verbose, } = config_ini_settings; - // get deployment.json variables - let deployment_json_settings = get_deployment_settings(&deployment_json_path)?; - let network_settings = NetworkSettings { ipv6_prefix, ipv6_address, @@ -87,14 +84,20 @@ pub fn main() -> Result<()> { ipv4_gateway, ipv4_prefix_length, domain, - mgmt_mac: deployment_json_settings.deployment.mgmt_mac, }; + // get deployment.json variables + let deployment_json_settings = get_deployment_settings(&deployment_json_path)?; + let logging = Logging { elasticsearch_hosts: deployment_json_settings.logging.hosts.to_string(), elasticsearch_tags: None, }; + let icos_dev_settings = ICOSDevSettings { + mgmt_mac: deployment_json_settings.deployment.mgmt_mac, + }; + let icos_settings = ICOSSettings { logging, nns_public_key_path: nns_public_key_path.to_path_buf(), @@ -106,6 +109,7 @@ pub fn main() -> Result<()> { ssh_authorized_keys_path: ssh_authorized_keys_path .exists() .then_some(ssh_authorized_keys_path), + icos_dev_settings, }; let setupos_settings = SetupOSSettings; diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 7562b55e12e..f293f8b187e 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -58,12 +58,12 @@ pub struct GuestOSSettings { /// When given, this provides the initial state of the registry. /// If not given, the node will fetch (initial) registry state from the NNS. pub ic_registry_local_store_path: Option, - pub guestos_dev: GuestosDevConfig, + pub guestos_dev_settings: GuestosDevSettings, } /// GuestOS development configuration. These settings are strictly used for development images. #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] -pub struct GuestosDevConfig { +pub struct GuestosDevSettings { pub backup_spool: Option, pub malicious_behavior: Option, pub query_stats_epoch_length: Option, @@ -93,7 +93,6 @@ pub struct NetworkSettings { pub ipv4_gateway: Option, pub ipv4_prefix_length: Option, pub domain: Option, - pub mgmt_mac: Option, } #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] @@ -114,6 +113,12 @@ pub struct ICOSSettings { /// backup and readonly can only be modified via an NNS proposal /// and are in place for subnet recovery or issue debugging purposes. pub ssh_authorized_keys_path: Option, + pub icos_dev_settings: ICOSDevSettings, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +pub struct ICOSDevSettings { + pub mgmt_mac: Option, } #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] From e1aaf28e0c991fdbfa28c1f40639be1465d4f2c7 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 25 Sep 2024 14:19:35 +0000 Subject: [PATCH 004/241] Update name of GuestOSDevSettings for consistency --- rs/ic_os/config/src/lib.rs | 4 ++-- rs/ic_os/config/src/types.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index a1bc0051f08..d51e9a774a3 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -47,7 +47,7 @@ mod tests { use super::*; use std::path::PathBuf; use types::{ - GuestOSConfig, GuestOSSettings, GuestosDevSettings, HostOSConfig, HostOSSettings, + GuestOSConfig, GuestOSDevSettings, GuestOSSettings, HostOSConfig, HostOSSettings, ICOSDevSettings, ICOSSettings, Logging, NetworkSettings, SetupOSConfig, SetupOSSettings, }; @@ -93,7 +93,7 @@ mod tests { ic_crypto_path: None, ic_state_path: None, ic_registry_local_store_path: None, - guestos_dev_settings: GuestosDevSettings::default(), + guestos_dev_settings: GuestOSDevSettings::default(), }; let setupos_config_struct = SetupOSConfig { diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index f293f8b187e..33caf0f3d7c 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -58,12 +58,12 @@ pub struct GuestOSSettings { /// When given, this provides the initial state of the registry. /// If not given, the node will fetch (initial) registry state from the NNS. pub ic_registry_local_store_path: Option, - pub guestos_dev_settings: GuestosDevSettings, + pub guestos_dev_settings: GuestOSDevSettings, } /// GuestOS development configuration. These settings are strictly used for development images. #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] -pub struct GuestosDevSettings { +pub struct GuestOSDevSettings { pub backup_spool: Option, pub malicious_behavior: Option, pub query_stats_epoch_length: Option, From 1723eb1cea731ce6f006917e4b31379ba4bf0e8a Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 25 Sep 2024 19:17:06 +0000 Subject: [PATCH 005/241] Update query_stats_epoch_length and BackupSpoolSettings to hold u64 --- rs/ic_os/config/src/types.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 33caf0f3d7c..628b7e6ce4e 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -66,7 +66,7 @@ pub struct GuestOSSettings { pub struct GuestOSDevSettings { pub backup_spool: Option, pub malicious_behavior: Option, - pub query_stats_epoch_length: Option, + pub query_stats_epoch_length: Option, pub bitcoind_addr: Option, pub jaeger_addr: Option, pub socks_proxy: Option, @@ -76,9 +76,9 @@ pub struct GuestOSDevSettings { #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] pub struct BackupSpoolSettings { /// The maximum age of any file or directory kept in the backup spool. - pub backup_retention_time_seconds: Option, + pub backup_retention_time_seconds: Option, /// The interval at which the backup spool directory will be scanned for files to delete. - pub backup_purging_interval_seconds: Option, + pub backup_purging_interval_seconds: Option, } #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] From 5e883bda7fac6ee206af0e3b3615036910d8c549 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Sep 2024 19:43:06 +0000 Subject: [PATCH 006/241] Remove repeat code --- ic-os/components/setupos-scripts/check-network.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index 79f16c68f97..8eb9932a420 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -172,16 +172,12 @@ function query_nns_nodes() { # When running against testnets, we need to ignore self signed certs # with `--insecure`. This check is only meant to confirm from SetupOS # that NNS urls are reachable, so we do not mind that it is "weak". - if curl --insecure --head --connect-timeout 3 --silent "${url}" >/dev/null 2>&1; then if curl --insecure --head --connect-timeout 3 --silent "${url}" >/dev/null 2>&1; then echo " okay: ${url}" success=true - success=true break else echo " fail: ${url}" - else - echo " fail: ${url}" fi done @@ -190,12 +186,6 @@ function query_nns_nodes() { else log_and_halt_installation_on_error "1" "Unable to query enough healthy NNS nodes." fi - - if $success; then - echo " success" - else - log_and_halt_installation_on_error "1" "Unable to query enough healthy NNS nodes." - fi } # Establish run order From 4517b6376d265161e0689fb9d5426dc6681bfc3f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Sep 2024 21:03:55 +0000 Subject: [PATCH 007/241] Revert "Merge branch 'master' into andrew/config-revamp-integration" This reverts commit cddb7f34fd1a42db20d2a225dfbc7eabbb947379, reversing changes made to 5e883bda7fac6ee206af0e3b3615036910d8c549. --- .../setup-hostname/hostos/setup-hostname.sh | 1 + .../guestos/guestos.xml.template | 2 +- .../hostos-scripts/guestos/qemu-cpu.xml | 2 +- .../hostos-scripts/libvirt/setup-libvirt.sh | 3 - ic-os/defs.bzl | 2 + ic-os/hostos/context/Dockerfile | 13 +++-- ic-os/hostos/context/docker-base.dev | 2 +- ic-os/hostos/context/docker-base.prod | 2 +- rs/ic_os/config/src/config_ini.rs | 55 +++++++++++++++---- rs/ic_os/config/src/lib.rs | 5 +- rs/ic_os/config/src/main.rs | 2 + rs/ic_os/config/src/types.rs | 5 +- rs/ic_os/network/src/lib.rs | 12 +++- 13 files changed, 78 insertions(+), 28 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh index 52322805d19..e4d35fbf0d6 100755 --- a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh @@ -61,6 +61,7 @@ function read_variables() { case "$key" in "ipv6_prefix") ipv6_prefix="${value}" ;; "ipv6_gateway") ipv6_gateway="${value}" ;; + "ipv6_address") ipv6_address="${value}" ;; "hostname") hostname="${value}" ;; esac done <"${CONFIG}" diff --git a/ic-os/components/hostos-scripts/guestos/guestos.xml.template b/ic-os/components/hostos-scripts/guestos/guestos.xml.template index bb58840fb0b..0e94d406d57 100755 --- a/ic-os/components/hostos-scripts/guestos/guestos.xml.template +++ b/ic-os/components/hostos-scripts/guestos/guestos.xml.template @@ -15,7 +15,7 @@ hvm - /usr/share/OVMF/OVMF_CODE_4M.fd + /usr/share/OVMF/OVMF_CODE.fd /var/lib/libvirt/qemu/nvram/guestos_VARS.fd diff --git a/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml b/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml index fd50c03a79e..3013b06373d 100644 --- a/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml +++ b/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml @@ -1 +1 @@ - + diff --git a/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh b/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh index c2b8b8d2017..5525702d98a 100755 --- a/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh +++ b/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh @@ -5,6 +5,3 @@ set -e # Create space for libvirt to manage its config mount --bind /run/libvirt /etc/libvirt - -# Set up log directory, because it will not create it alone -mkdir -p /var/log/libvirt/qemu diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl index 5f2a4384cfd..5a4e614c977 100644 --- a/ic-os/defs.bzl +++ b/ic-os/defs.bzl @@ -148,6 +148,8 @@ def icos_build( "/run", "/boot", "/var", + "/usr/lib/firmware/brcm/brcmfmac43430a0-sdio.ONDA-V80 PLUS.txt", + "/usr/lib/firmware/brcm/brcmfmac43455-sdio.MINIX-NEO Z83-4.txt", "/usr/lib/firmware/brcm/brcmfmac43241b4-sdio.Intel Corp.-VALLEYVIEW C0 PLATFORM.txt.zst", "/usr/lib/firmware/brcm/brcmfmac43340-sdio.ASUSTeK COMPUTER INC.-TF103CE.txt.zst", "/usr/lib/firmware/brcm/brcmfmac43362-sdio.ASUSTeK COMPUTER INC.-ME176C.txt.zst", diff --git a/ic-os/hostos/context/Dockerfile b/ic-os/hostos/context/Dockerfile index 621c23eeda6..5b3374fe043 100644 --- a/ic-os/hostos/context/Dockerfile +++ b/ic-os/hostos/context/Dockerfile @@ -29,7 +29,7 @@ RUN sed -e '/.*pam_motd.so.*/d' -i /etc/pam.d/login && \ # but this is per system (so backups are not persisted across upgrades) # and thus not very useful, and /etc is read-only. # So simply suppress generating backups. -RUN sed -e 's/\(# \)\?\(backup *= *\)[01]/\20/' -e 's/\(# \)\?\(archive *= *\)[01]/\20/' -i /etc/lvm/lvm.conf +RUN sed -e 's/\(backup *= *\)1/\10/' -e 's/\(archive *= *\)1/\10/' -i /etc/lvm/lvm.conf # Deactivate systemd userdb. We don't use it. RUN sed -e 's/ *systemd//' -i /etc/nsswitch.conf @@ -39,7 +39,7 @@ RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 # Clear files that may lead to indeterministic build. RUN apt-get clean && \ - find /usr/lib/python3.12 -name "*.pyc" | xargs rm && \ + find /usr/lib/python3.8 -name "*.pyc" | xargs rm && \ find /usr/lib/python3 -name "*.pyc" | xargs rm && \ find /usr/share/python3 -name "*.pyc" | xargs rm && \ truncate --size 0 /etc/machine-id @@ -91,7 +91,6 @@ RUN systemctl enable \ chrony \ libvirtd \ nftables \ - ssh \ systemd-journal-gatewayd \ systemd-networkd \ systemd-networkd-wait-online \ @@ -118,8 +117,14 @@ RUN rm -rf \ /usr/local/share/qemu/edk2-arm-code.fd \ /usr/local/share/qemu/edk2-arm-vars.fd +# Add user/group entries specified here: /usr/lib/sysusers.d/systemd.conf E.g., systemd-timesync/coredump +RUN systemd-sysusers && \ + # Fix reproducibility issue. Notes in hostos/context/Dockerfile + usermod -p '!!' systemd-timesync && \ + usermod -p '!!' systemd-coredump + # Set /bin/sh to point to /bin/bash instead of the default /bin/dash -RUN ln -sf bash /usr/bin/sh +RUN echo "set dash/sh false" | debconf-communicate && dpkg-reconfigure -fnoninteractive dash # Group accounts to which parts of the runtime state are assigned such that # user accounts can be granted individual access rights. diff --git a/ic-os/hostos/context/docker-base.dev b/ic-os/hostos/context/docker-base.dev index 6b56f76b0ae..cd5decc792b 100644 --- a/ic-os/hostos/context/docker-base.dev +++ b/ic-os/hostos/context/docker-base.dev @@ -1 +1 @@ -ghcr.io/dfinity/hostos-base-dev@sha256:e133ba80d1d291fff89fe9e60f3fbf9285095d9c15a51a1297e314d0f9fe837c +ghcr.io/dfinity/hostos-base-dev@sha256:a6e8e7ab7abf682c80dfd717d36ac027213f757ddad415c66080ac4314639590 diff --git a/ic-os/hostos/context/docker-base.prod b/ic-os/hostos/context/docker-base.prod index a18a83e2197..0c1ef008b24 100644 --- a/ic-os/hostos/context/docker-base.prod +++ b/ic-os/hostos/context/docker-base.prod @@ -1 +1 @@ -ghcr.io/dfinity/hostos-base@sha256:13ae203beb66cdb2ac198ea2441e82b6fb36b93c4e337ccbc32383de49ad7f88 +ghcr.io/dfinity/hostos-base@sha256:5e93fb6cadecd22b838a8e442ed88d3c77ac5626651ee139559150d2a77a6743 diff --git a/rs/ic_os/config/src/config_ini.rs b/rs/ic_os/config/src/config_ini.rs index 4bc6c92219c..d90a77d95be 100644 --- a/rs/ic_os/config/src/config_ini.rs +++ b/rs/ic_os/config/src/config_ini.rs @@ -9,7 +9,8 @@ use anyhow::{Context, Result}; pub type ConfigMap = HashMap; pub struct ConfigIniSettings { - pub ipv6_prefix: String, + pub ipv6_prefix: Option, + pub ipv6_address: Option, pub ipv6_prefix_length: u8, pub ipv6_gateway: Ipv6Addr, pub ipv4_address: Option, @@ -30,18 +31,34 @@ pub fn get_config_ini_settings(config_file_path: &Path) -> Result() + .context(format!("Invalid IPv6 address: {}", address)) + }) + .transpose()?; + + if ipv6_address.is_none() && ipv6_prefix.is_none() { + bail!("Missing config parameter: need at least one of ipv6_prefix or ipv6_address"); + } + let ipv6_gateway = config_map .get("ipv6_gateway") .context("Missing config parameter: ipv6_gateway")? @@ -90,6 +107,7 @@ pub fn get_config_ini_settings(config_file_path: &Path) -> Result()? + ); assert_eq!( config_ini_settings.ipv6_gateway, "2a00:fb01:400:200::1".parse::()? @@ -268,6 +291,16 @@ mod tests { assert_eq!(config_ini_settings.domain, Some("example.com".to_string())); assert!(!config_ini_settings.verbose); + // Test ipv6_address without ipv6_prefix_length length + let mut temp_file = NamedTempFile::new()?; + writeln!(temp_file, "ipv6_address=2a00:fb01:400:200::")?; + let config_ini_settings = get_config_ini_settings(temp_file_path)?; + assert_eq!( + config_ini_settings.ipv6_address.unwrap(), + "2a00:fb01:400:200::".parse::()? + ); + assert_eq!(config_ini_settings.ipv6_prefix_length, 64); + // Test missing ipv6 let mut temp_file = NamedTempFile::new()?; writeln!(temp_file, "ipv4_address=212.71.124.178")?; @@ -278,7 +311,7 @@ mod tests { let result = get_config_ini_settings(temp_file_path); assert!(result.is_err()); - // Test invalid IPv6 prefix + // Test invalid IPv6 address let mut temp_file = NamedTempFile::new()?; writeln!(temp_file, "ipv6_prefix=invalid_ipv6_prefix")?; writeln!(temp_file, "ipv6_gateway=2001:db8:85a3:0000::1")?; @@ -290,7 +323,7 @@ mod tests { let result = get_config_ini_settings(temp_file_path); assert!(result.is_err()); - // Test missing prefix + // Test missing prefix and address let mut temp_file = NamedTempFile::new()?; writeln!(temp_file, "ipv6_gateway=2001:db8:85a3:0000::1")?; let result = get_config_ini_settings(temp_file_path); diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 2e4cf733440..c49f689cb53 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -54,9 +54,10 @@ mod tests { #[test] fn test_serialize_and_deserialize() { let network_settings = NetworkSettings { - ipv6_prefix: "2a00:fb01:400:200".to_string(), + ipv6_prefix: None, + ipv6_address: None, ipv6_prefix_length: 64_u8, - ipv6_gateway: "2a00:fb01:400:200::1".parse().unwrap(), + ipv6_gateway: "2001:db8::1".parse().unwrap(), ipv4_address: None, ipv4_gateway: None, ipv4_prefix_length: None, diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index ba0dde4f2b3..65e825ee95c 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -65,6 +65,7 @@ pub fn main() -> Result<()> { let config_ini_settings = get_config_ini_settings(&config_ini_path)?; let ConfigIniSettings { ipv6_prefix, + ipv6_address, ipv6_prefix_length, ipv6_gateway, ipv4_address, @@ -79,6 +80,7 @@ pub fn main() -> Result<()> { let network_settings = NetworkSettings { ipv6_prefix, + ipv6_address, ipv6_prefix_length, ipv6_gateway, ipv4_address, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 5940a7dc1ea..7562b55e12e 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -83,7 +83,10 @@ pub struct BackupSpoolSettings { #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] pub struct NetworkSettings { - pub ipv6_prefix: String, + // Config.ini can specify ipv6_prefix and ipv6_gateway, or just an ipv6_address. + // ipv6_address takes precedence. Some tests provide only ipv6_address. + pub ipv6_prefix: Option, + pub ipv6_address: Option, pub ipv6_prefix_length: u8, pub ipv6_gateway: Ipv6Addr, pub ipv4_address: Option, diff --git a/rs/ic_os/network/src/lib.rs b/rs/ic_os/network/src/lib.rs index 8538df3eab4..caa65ca1642 100644 --- a/rs/ic_os/network/src/lib.rs +++ b/rs/ic_os/network/src/lib.rs @@ -23,10 +23,16 @@ pub fn generate_network_config( node_type: NodeType, output_directory: &Path, ) -> Result<()> { - let deployment_name = deployment_name - .context("Error: Deployment name not found when attempting to generate mac address")?; + if let Some(address) = network_settings.ipv6_address { + eprintln!("Found ipv6 address in config"); + return generate_systemd_config_files(output_directory, network_settings, None, &address); + }; - let mac = generate_mac_address(deployment_name, &node_type, mgmt_mac)?; + let mac = generate_mac_address( + deployment_name, + &node_type, + network_settings.mgmt_mac.as_deref(), + )?; eprintln!("Using generated mac (unformatted) {}", mac.get()); eprintln!("Generating ipv6 address"); From 4cd274284405678d9f955261dec0dd14bec77734 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Sep 2024 21:05:39 +0000 Subject: [PATCH 008/241] Revert "Revert "Merge branch 'master' into andrew/config-revamp-integration"" This reverts commit 4517b6376d265161e0689fb9d5426dc6681bfc3f. --- .../setup-hostname/hostos/setup-hostname.sh | 1 - .../guestos/guestos.xml.template | 2 +- .../hostos-scripts/guestos/qemu-cpu.xml | 2 +- .../hostos-scripts/libvirt/setup-libvirt.sh | 3 + ic-os/defs.bzl | 2 - ic-os/hostos/context/Dockerfile | 13 ++--- ic-os/hostos/context/docker-base.dev | 2 +- ic-os/hostos/context/docker-base.prod | 2 +- rs/ic_os/config/src/config_ini.rs | 55 ++++--------------- rs/ic_os/config/src/lib.rs | 5 +- rs/ic_os/config/src/main.rs | 2 - rs/ic_os/config/src/types.rs | 5 +- rs/ic_os/network/src/lib.rs | 12 +--- 13 files changed, 28 insertions(+), 78 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh index e4d35fbf0d6..52322805d19 100755 --- a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh @@ -61,7 +61,6 @@ function read_variables() { case "$key" in "ipv6_prefix") ipv6_prefix="${value}" ;; "ipv6_gateway") ipv6_gateway="${value}" ;; - "ipv6_address") ipv6_address="${value}" ;; "hostname") hostname="${value}" ;; esac done <"${CONFIG}" diff --git a/ic-os/components/hostos-scripts/guestos/guestos.xml.template b/ic-os/components/hostos-scripts/guestos/guestos.xml.template index 0e94d406d57..bb58840fb0b 100755 --- a/ic-os/components/hostos-scripts/guestos/guestos.xml.template +++ b/ic-os/components/hostos-scripts/guestos/guestos.xml.template @@ -15,7 +15,7 @@ hvm - /usr/share/OVMF/OVMF_CODE.fd + /usr/share/OVMF/OVMF_CODE_4M.fd /var/lib/libvirt/qemu/nvram/guestos_VARS.fd diff --git a/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml b/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml index 3013b06373d..fd50c03a79e 100644 --- a/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml +++ b/ic-os/components/hostos-scripts/guestos/qemu-cpu.xml @@ -1 +1 @@ - + diff --git a/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh b/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh index 5525702d98a..c2b8b8d2017 100755 --- a/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh +++ b/ic-os/components/hostos-scripts/libvirt/setup-libvirt.sh @@ -5,3 +5,6 @@ set -e # Create space for libvirt to manage its config mount --bind /run/libvirt /etc/libvirt + +# Set up log directory, because it will not create it alone +mkdir -p /var/log/libvirt/qemu diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl index 5a4e614c977..5f2a4384cfd 100644 --- a/ic-os/defs.bzl +++ b/ic-os/defs.bzl @@ -148,8 +148,6 @@ def icos_build( "/run", "/boot", "/var", - "/usr/lib/firmware/brcm/brcmfmac43430a0-sdio.ONDA-V80 PLUS.txt", - "/usr/lib/firmware/brcm/brcmfmac43455-sdio.MINIX-NEO Z83-4.txt", "/usr/lib/firmware/brcm/brcmfmac43241b4-sdio.Intel Corp.-VALLEYVIEW C0 PLATFORM.txt.zst", "/usr/lib/firmware/brcm/brcmfmac43340-sdio.ASUSTeK COMPUTER INC.-TF103CE.txt.zst", "/usr/lib/firmware/brcm/brcmfmac43362-sdio.ASUSTeK COMPUTER INC.-ME176C.txt.zst", diff --git a/ic-os/hostos/context/Dockerfile b/ic-os/hostos/context/Dockerfile index 5b3374fe043..621c23eeda6 100644 --- a/ic-os/hostos/context/Dockerfile +++ b/ic-os/hostos/context/Dockerfile @@ -29,7 +29,7 @@ RUN sed -e '/.*pam_motd.so.*/d' -i /etc/pam.d/login && \ # but this is per system (so backups are not persisted across upgrades) # and thus not very useful, and /etc is read-only. # So simply suppress generating backups. -RUN sed -e 's/\(backup *= *\)1/\10/' -e 's/\(archive *= *\)1/\10/' -i /etc/lvm/lvm.conf +RUN sed -e 's/\(# \)\?\(backup *= *\)[01]/\20/' -e 's/\(# \)\?\(archive *= *\)[01]/\20/' -i /etc/lvm/lvm.conf # Deactivate systemd userdb. We don't use it. RUN sed -e 's/ *systemd//' -i /etc/nsswitch.conf @@ -39,7 +39,7 @@ RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 # Clear files that may lead to indeterministic build. RUN apt-get clean && \ - find /usr/lib/python3.8 -name "*.pyc" | xargs rm && \ + find /usr/lib/python3.12 -name "*.pyc" | xargs rm && \ find /usr/lib/python3 -name "*.pyc" | xargs rm && \ find /usr/share/python3 -name "*.pyc" | xargs rm && \ truncate --size 0 /etc/machine-id @@ -91,6 +91,7 @@ RUN systemctl enable \ chrony \ libvirtd \ nftables \ + ssh \ systemd-journal-gatewayd \ systemd-networkd \ systemd-networkd-wait-online \ @@ -117,14 +118,8 @@ RUN rm -rf \ /usr/local/share/qemu/edk2-arm-code.fd \ /usr/local/share/qemu/edk2-arm-vars.fd -# Add user/group entries specified here: /usr/lib/sysusers.d/systemd.conf E.g., systemd-timesync/coredump -RUN systemd-sysusers && \ - # Fix reproducibility issue. Notes in hostos/context/Dockerfile - usermod -p '!!' systemd-timesync && \ - usermod -p '!!' systemd-coredump - # Set /bin/sh to point to /bin/bash instead of the default /bin/dash -RUN echo "set dash/sh false" | debconf-communicate && dpkg-reconfigure -fnoninteractive dash +RUN ln -sf bash /usr/bin/sh # Group accounts to which parts of the runtime state are assigned such that # user accounts can be granted individual access rights. diff --git a/ic-os/hostos/context/docker-base.dev b/ic-os/hostos/context/docker-base.dev index cd5decc792b..6b56f76b0ae 100644 --- a/ic-os/hostos/context/docker-base.dev +++ b/ic-os/hostos/context/docker-base.dev @@ -1 +1 @@ -ghcr.io/dfinity/hostos-base-dev@sha256:a6e8e7ab7abf682c80dfd717d36ac027213f757ddad415c66080ac4314639590 +ghcr.io/dfinity/hostos-base-dev@sha256:e133ba80d1d291fff89fe9e60f3fbf9285095d9c15a51a1297e314d0f9fe837c diff --git a/ic-os/hostos/context/docker-base.prod b/ic-os/hostos/context/docker-base.prod index 0c1ef008b24..a18a83e2197 100644 --- a/ic-os/hostos/context/docker-base.prod +++ b/ic-os/hostos/context/docker-base.prod @@ -1 +1 @@ -ghcr.io/dfinity/hostos-base@sha256:5e93fb6cadecd22b838a8e442ed88d3c77ac5626651ee139559150d2a77a6743 +ghcr.io/dfinity/hostos-base@sha256:13ae203beb66cdb2ac198ea2441e82b6fb36b93c4e337ccbc32383de49ad7f88 diff --git a/rs/ic_os/config/src/config_ini.rs b/rs/ic_os/config/src/config_ini.rs index d90a77d95be..4bc6c92219c 100644 --- a/rs/ic_os/config/src/config_ini.rs +++ b/rs/ic_os/config/src/config_ini.rs @@ -9,8 +9,7 @@ use anyhow::{Context, Result}; pub type ConfigMap = HashMap; pub struct ConfigIniSettings { - pub ipv6_prefix: Option, - pub ipv6_address: Option, + pub ipv6_prefix: String, pub ipv6_prefix_length: u8, pub ipv6_gateway: Ipv6Addr, pub ipv4_address: Option, @@ -31,34 +30,18 @@ pub fn get_config_ini_settings(config_file_path: &Path) -> Result() - .context(format!("Invalid IPv6 address: {}", address)) - }) - .transpose()?; - - if ipv6_address.is_none() && ipv6_prefix.is_none() { - bail!("Missing config parameter: need at least one of ipv6_prefix or ipv6_address"); - } - let ipv6_gateway = config_map .get("ipv6_gateway") .context("Missing config parameter: ipv6_gateway")? @@ -107,7 +90,6 @@ pub fn get_config_ini_settings(config_file_path: &Path) -> Result()? - ); assert_eq!( config_ini_settings.ipv6_gateway, "2a00:fb01:400:200::1".parse::()? @@ -291,16 +268,6 @@ mod tests { assert_eq!(config_ini_settings.domain, Some("example.com".to_string())); assert!(!config_ini_settings.verbose); - // Test ipv6_address without ipv6_prefix_length length - let mut temp_file = NamedTempFile::new()?; - writeln!(temp_file, "ipv6_address=2a00:fb01:400:200::")?; - let config_ini_settings = get_config_ini_settings(temp_file_path)?; - assert_eq!( - config_ini_settings.ipv6_address.unwrap(), - "2a00:fb01:400:200::".parse::()? - ); - assert_eq!(config_ini_settings.ipv6_prefix_length, 64); - // Test missing ipv6 let mut temp_file = NamedTempFile::new()?; writeln!(temp_file, "ipv4_address=212.71.124.178")?; @@ -311,7 +278,7 @@ mod tests { let result = get_config_ini_settings(temp_file_path); assert!(result.is_err()); - // Test invalid IPv6 address + // Test invalid IPv6 prefix let mut temp_file = NamedTempFile::new()?; writeln!(temp_file, "ipv6_prefix=invalid_ipv6_prefix")?; writeln!(temp_file, "ipv6_gateway=2001:db8:85a3:0000::1")?; @@ -323,7 +290,7 @@ mod tests { let result = get_config_ini_settings(temp_file_path); assert!(result.is_err()); - // Test missing prefix and address + // Test missing prefix let mut temp_file = NamedTempFile::new()?; writeln!(temp_file, "ipv6_gateway=2001:db8:85a3:0000::1")?; let result = get_config_ini_settings(temp_file_path); diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index c49f689cb53..2e4cf733440 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -54,10 +54,9 @@ mod tests { #[test] fn test_serialize_and_deserialize() { let network_settings = NetworkSettings { - ipv6_prefix: None, - ipv6_address: None, + ipv6_prefix: "2a00:fb01:400:200".to_string(), ipv6_prefix_length: 64_u8, - ipv6_gateway: "2001:db8::1".parse().unwrap(), + ipv6_gateway: "2a00:fb01:400:200::1".parse().unwrap(), ipv4_address: None, ipv4_gateway: None, ipv4_prefix_length: None, diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 65e825ee95c..ba0dde4f2b3 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -65,7 +65,6 @@ pub fn main() -> Result<()> { let config_ini_settings = get_config_ini_settings(&config_ini_path)?; let ConfigIniSettings { ipv6_prefix, - ipv6_address, ipv6_prefix_length, ipv6_gateway, ipv4_address, @@ -80,7 +79,6 @@ pub fn main() -> Result<()> { let network_settings = NetworkSettings { ipv6_prefix, - ipv6_address, ipv6_prefix_length, ipv6_gateway, ipv4_address, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 7562b55e12e..5940a7dc1ea 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -83,10 +83,7 @@ pub struct BackupSpoolSettings { #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] pub struct NetworkSettings { - // Config.ini can specify ipv6_prefix and ipv6_gateway, or just an ipv6_address. - // ipv6_address takes precedence. Some tests provide only ipv6_address. - pub ipv6_prefix: Option, - pub ipv6_address: Option, + pub ipv6_prefix: String, pub ipv6_prefix_length: u8, pub ipv6_gateway: Ipv6Addr, pub ipv4_address: Option, diff --git a/rs/ic_os/network/src/lib.rs b/rs/ic_os/network/src/lib.rs index caa65ca1642..8538df3eab4 100644 --- a/rs/ic_os/network/src/lib.rs +++ b/rs/ic_os/network/src/lib.rs @@ -23,16 +23,10 @@ pub fn generate_network_config( node_type: NodeType, output_directory: &Path, ) -> Result<()> { - if let Some(address) = network_settings.ipv6_address { - eprintln!("Found ipv6 address in config"); - return generate_systemd_config_files(output_directory, network_settings, None, &address); - }; + let deployment_name = deployment_name + .context("Error: Deployment name not found when attempting to generate mac address")?; - let mac = generate_mac_address( - deployment_name, - &node_type, - network_settings.mgmt_mac.as_deref(), - )?; + let mac = generate_mac_address(deployment_name, &node_type, mgmt_mac)?; eprintln!("Using generated mac (unformatted) {}", mac.get()); eprintln!("Generating ipv6 address"); From d622da8334a79f0fa494ee1ce55dcaf4701d3399 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Sep 2024 21:13:35 +0000 Subject: [PATCH 009/241] Fix merge errors --- rs/ic_os/network/src/lib.rs | 16 +++++++--------- rs/ic_os/os_tools/hostos_tool/src/main.rs | 10 ++-------- rs/ic_os/os_tools/setupos_tool/src/main.rs | 6 ++---- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/rs/ic_os/network/src/lib.rs b/rs/ic_os/network/src/lib.rs index 8538df3eab4..3bfa6b0529c 100644 --- a/rs/ic_os/network/src/lib.rs +++ b/rs/ic_os/network/src/lib.rs @@ -1,6 +1,6 @@ use std::path::Path; -use anyhow::{Context, Result}; +use anyhow::Result; use crate::mac_address::generate_mac_address; use crate::node_type::NodeType; @@ -23,17 +23,15 @@ pub fn generate_network_config( node_type: NodeType, output_directory: &Path, ) -> Result<()> { - let deployment_name = deployment_name - .context("Error: Deployment name not found when attempting to generate mac address")?; - - let mac = generate_mac_address(deployment_name, &node_type, mgmt_mac)?; + let mac = generate_mac_address( + deployment_name, + &node_type, + network_settings.mgmt_mac.as_deref(), + )?; eprintln!("Using generated mac (unformatted) {}", mac.get()); eprintln!("Generating ipv6 address"); - let ipv6_prefix = network_settings - .ipv6_prefix - .clone() - .context("ipv6_prefix required in config to generate ipv6 address")?; + let ipv6_prefix = network_settings.ipv6_prefix.clone(); let ipv6_address = generate_ipv6_address(&ipv6_prefix, &mac)?; eprintln!("Using ipv6 address: {}", ipv6_address); diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 20442549009..90c60d1b042 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -1,6 +1,6 @@ use std::path::Path; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use config::config_ini::get_config_ini_settings; @@ -67,7 +67,6 @@ pub fn main() -> Result<()> { // and we won't need to read config.ini and deployment.json directly. let network_settings = NetworkSettings { ipv6_prefix: config_ini_settings.ipv6_prefix, - ipv6_address: config_ini_settings.ipv6_address, ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, ipv6_gateway: config_ini_settings.ipv6_gateway, ipv4_address: config_ini_settings.ipv4_address, @@ -97,7 +96,6 @@ pub fn main() -> Result<()> { // and we won't need to read config.ini and deployment.json directly. let network_settings = NetworkSettings { ipv6_prefix: config_ini_settings.ipv6_prefix, - ipv6_address: config_ini_settings.ipv6_address, ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, ipv6_gateway: config_ini_settings.ipv6_gateway, ipv4_address: config_ini_settings.ipv4_address, @@ -114,10 +112,7 @@ pub fn main() -> Result<()> { &node_type, network_settings.mgmt_mac.as_deref(), )?; - let ipv6_prefix = network_settings - .ipv6_prefix - .context("ipv6_prefix required in config to generate ipv6 address")?; - let ipv6_address = generate_ipv6_address(&ipv6_prefix, &mac)?; + let ipv6_address = generate_ipv6_address(&network_settings.ipv6_prefix, &mac)?; println!( "{}", to_cidr(ipv6_address, network_settings.ipv6_prefix_length) @@ -136,7 +131,6 @@ pub fn main() -> Result<()> { // and we won't need to read config.ini and deployment.json directly. let network_settings = NetworkSettings { ipv6_prefix: config_ini_settings.ipv6_prefix, - ipv6_address: config_ini_settings.ipv6_address, ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, ipv6_gateway: config_ini_settings.ipv6_gateway, ipv4_address: config_ini_settings.ipv4_address, diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 77063696d0f..3d2994f26c2 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -78,10 +78,8 @@ pub fn main() -> Result<()> { &node_type, setup_config.network_settings.mgmt_mac.as_deref(), )?; - let ipv6_prefix = setup_config.network_settings.ipv6_prefix.ok_or_else(|| { - anyhow!("ipv6_prefix required in config to generate ipv6 address") - })?; - let ipv6_address = generate_ipv6_address(&ipv6_prefix, &mac)?; + let ipv6_address = + generate_ipv6_address(&setup_config.network_settings.ipv6_prefix, &mac)?; println!( "{}", to_cidr( From 514c2315b1769ca7241ad804accd51329264dc3c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Sep 2024 21:17:13 +0000 Subject: [PATCH 010/241] Remove unnecessary ipv6_prefix declaration --- rs/ic_os/network/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rs/ic_os/network/src/lib.rs b/rs/ic_os/network/src/lib.rs index 3bfa6b0529c..ecd945d9c53 100644 --- a/rs/ic_os/network/src/lib.rs +++ b/rs/ic_os/network/src/lib.rs @@ -31,8 +31,7 @@ pub fn generate_network_config( eprintln!("Using generated mac (unformatted) {}", mac.get()); eprintln!("Generating ipv6 address"); - let ipv6_prefix = network_settings.ipv6_prefix.clone(); - let ipv6_address = generate_ipv6_address(&ipv6_prefix, &mac)?; + let ipv6_address = generate_ipv6_address(&network_settings.ipv6_prefix, &mac)?; eprintln!("Using ipv6 address: {}", ipv6_address); let formatted_mac = FormattedMacAddress::from(&mac); From 40480c5f6ec6d931ab8dadbf1eca4aaaa60b797c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 30 Sep 2024 15:21:32 +0000 Subject: [PATCH 011/241] Fix ownership error --- rs/ic_os/os_tools/setupos_tool/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 1f3b8f8be10..dec348a556f 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -60,7 +60,7 @@ pub fn main() -> Result<()> { &setup_config.network_settings ); - let mgmt_mac = match setup_config.network_settings.mgmt_mac { + let mgmt_mac = match setup_config.network_settings.mgmt_mac.as_ref() { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( From 168f175ea60f237fabccba065608c6b0c06ba61e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 30 Sep 2024 15:25:58 +0000 Subject: [PATCH 012/241] Re-add node_type parsing and fix formatting --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 2 +- rs/ic_os/os_tools/setupos_tool/src/main.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index dc873d01cfb..d072d0a524f 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -123,6 +123,7 @@ pub fn main() -> Result<()> { }; eprintln!("Network settings config: {:?}", &network_settings); + let node_type = node_type.parse::()?; let mgmt_mac = match network_settings.mgmt_mac.as_ref() { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; @@ -171,7 +172,6 @@ pub fn main() -> Result<()> { eprintln!("Network settings config: {:?}", &network_settings); let node_type = node_type.parse::()?; - let mgmt_mac = match network_settings.mgmt_mac.as_ref() { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index dec348a556f..e53d6cf220e 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -94,7 +94,6 @@ pub fn main() -> Result<()> { ); let node_type = node_type.parse::()?; - let mgmt_mac = match setup_config.network_settings.mgmt_mac { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; From 881e95e5c1dec714b534a1cb1b220ed55ffbc0c5 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 30 Sep 2024 17:47:29 +0000 Subject: [PATCH 013/241] Create NetworkSettings enums --- rs/ic_os/config/src/lib.rs | 19 +++++++-------- rs/ic_os/config/src/main.rs | 38 +++++++++++++++++++----------- rs/ic_os/config/src/types.rs | 45 +++++++++++++++++++++++++++--------- 3 files changed, 67 insertions(+), 35 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index c75ebb58d5c..d10ef35b8e4 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -46,21 +46,18 @@ pub fn deserialize_config Deserialize<'de>>(file_path: &str) -> Resu mod tests { use super::*; use std::path::PathBuf; - use types::{ - GuestOSConfig, GuestOSDevSettings, GuestOSSettings, HostOSConfig, HostOSSettings, - ICOSDevSettings, ICOSSettings, Logging, NetworkSettings, SetupOSConfig, SetupOSSettings, - }; + use types::*; #[test] fn test_serialize_and_deserialize() { + let ipv6_config = Ipv6Config::Deterministic(DeterministicIpv6Config { + prefix: "2a00:fb01:400:200".to_string(), + prefix_length: 64_u8, + gateway: "2a00:fb01:400:200::1".parse().unwrap(), + }); let network_settings = NetworkSettings { - ipv6_prefix: "2a00:fb01:400:200".to_string(), - ipv6_prefix_length: 64_u8, - ipv6_gateway: "2a00:fb01:400:200::1".parse().unwrap(), - ipv4_address: None, - ipv4_gateway: None, - ipv4_prefix_length: None, - domain: None, + ipv6_config, + ipv4_config: None, }; let logging = Logging { elasticsearch_hosts: [ diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 5e7b1ac54ba..ca8c3ebe86b 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -6,10 +6,7 @@ use config::serialize_and_write_config; use std::fs::File; use std::path::{Path, PathBuf}; -use config::types::{ - GuestOSSettings, HostOSConfig, HostOSSettings, ICOSDevSettings, ICOSSettings, Logging, - NetworkSettings, SetupOSConfig, SetupOSSettings, -}; +use config::types::*; #[derive(Subcommand)] pub enum Commands { @@ -62,7 +59,6 @@ pub fn main() -> Result<()> { setupos_config_json_path, }) => { // get config.ini settings - let config_ini_settings = get_config_ini_settings(&config_ini_path)?; let ConfigIniSettings { ipv6_prefix, ipv6_prefix_length, @@ -72,16 +68,32 @@ pub fn main() -> Result<()> { ipv4_prefix_length, domain, verbose, - } = config_ini_settings; + } = get_config_ini_settings(&config_ini_path)?; + + // create NetworkSettings + let deterministic_config = DeterministicIpv6Config { + prefix: ipv6_prefix, + prefix_length: ipv6_prefix_length, + gateway: ipv6_gateway, + }; + + let ipv4_config = + if let (Some(address), Some(gateway), Some(prefix_length), Some(domain)) = + (ipv4_address, ipv4_gateway, ipv4_prefix_length, domain) + { + Some(Ipv4Config { + address, + gateway, + prefix_length, + domain, + }) + } else { + None + }; let network_settings = NetworkSettings { - ipv6_prefix, - ipv6_prefix_length, - ipv6_gateway, - ipv4_address, - ipv4_gateway, - ipv4_prefix_length, - domain, + ipv6_config: Ipv6Config::Deterministic(deterministic_config), + ipv4_config, }; // get deployment.json variables diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 975b1093f2f..96ee595d615 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -81,17 +81,6 @@ pub struct BackupSpoolSettings { pub backup_purging_interval_seconds: Option, } -#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] -pub struct NetworkSettings { - pub ipv6_prefix: String, - pub ipv6_prefix_length: u8, - pub ipv6_gateway: Ipv6Addr, - pub ipv4_address: Option, - pub ipv4_gateway: Option, - pub ipv4_prefix_length: Option, - pub domain: Option, -} - #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] pub struct ICOSSettings { pub logging: Logging, @@ -125,3 +114,37 @@ pub struct Logging { /// Space-separated list of tags to apply to exported log records. pub elasticsearch_tags: Option, } + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +pub struct NetworkSettings { + pub ipv6_config: Ipv6Config, + pub ipv4_config: Option, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +pub struct Ipv4Config { + pub address: Ipv4Addr, + pub gateway: Ipv4Addr, + pub prefix_length: u8, + pub domain: String, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +pub enum Ipv6Config { + Deterministic(DeterministicIpv6Config), + Fixed(FixedIpv6Config), + RouterAdvertisement, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +pub struct DeterministicIpv6Config { + pub prefix: String, + pub prefix_length: u8, + pub gateway: Ipv6Addr, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +pub struct FixedIpv6Config { + pub address: Ipv6Addr, + pub gateway: Ipv6Addr, +} From 26324e44385e9017d7c8730923c81bd5fbc0170e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 30 Sep 2024 17:53:44 +0000 Subject: [PATCH 014/241] Print SetupOSConfig --- rs/ic_os/config/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index ca8c3ebe86b..10f989f04dc 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -143,6 +143,7 @@ pub fn main() -> Result<()> { hostos_settings, guestos_settings, }; + println!("SetupOSConfig: {:?}", setupos_config); let setupos_config_json_path = Path::new(&setupos_config_json_path); serialize_and_write_config(setupos_config_json_path, &setupos_config)?; From c3dd639bb1baec0d85204644a5ec8b3999d8ffa1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 2 Oct 2024 22:46:58 +0000 Subject: [PATCH 015/241] Fix errors after merging config tool refactor --- rs/ic_os/network/src/lib.rs | 31 ++++--- rs/ic_os/network/src/systemd.rs | 17 ++-- rs/ic_os/os_tools/hostos_tool/src/main.rs | 97 ++++++++++++++-------- rs/ic_os/os_tools/setupos_tool/src/main.rs | 30 ++++--- 4 files changed, 107 insertions(+), 68 deletions(-) diff --git a/rs/ic_os/network/src/lib.rs b/rs/ic_os/network/src/lib.rs index 081d2cb9fe4..01147f7a06e 100644 --- a/rs/ic_os/network/src/lib.rs +++ b/rs/ic_os/network/src/lib.rs @@ -1,10 +1,10 @@ use std::path::Path; -use anyhow::Result; +use anyhow::{anyhow, Result}; use crate::mac_address::UnformattedMacAddress; use crate::systemd::generate_systemd_config_files; -use config::types::NetworkSettings; +use config::types::{Ipv6Config, NetworkSettings}; use ipv6::generate_ipv6_address; use mac_address::FormattedMacAddress; @@ -22,14 +22,23 @@ pub fn generate_network_config( output_directory: &Path, ) -> Result<()> { eprintln!("Generating ipv6 address"); - let ipv6_address = generate_ipv6_address(&network_settings.ipv6_prefix, &generated_mac)?; - eprintln!("Using ipv6 address: {}", ipv6_address); - let formatted_mac = FormattedMacAddress::from(&generated_mac); - generate_systemd_config_files( - output_directory, - network_settings, - Some(&formatted_mac), - &ipv6_address, - ) + match &network_settings.ipv6_config { + Ipv6Config::RouterAdvertisement => { + Err(anyhow!("IC-OS router advertisement is not yet supported")) + } + Ipv6Config::Fixed(_) => Err(anyhow!("Fixed IP configuration is not yet supported")), + Ipv6Config::Deterministic(ipv6_config) => { + let ipv6_address = generate_ipv6_address(&ipv6_config.prefix, &generated_mac)?; + eprintln!("Using ipv6 address: {}", ipv6_address); + + let formatted_mac = FormattedMacAddress::from(&generated_mac); + generate_systemd_config_files( + output_directory, + ipv6_config, + Some(&formatted_mac), + &ipv6_address, + ) + } + } } diff --git a/rs/ic_os/network/src/systemd.rs b/rs/ic_os/network/src/systemd.rs index bd58f0f2916..e2d49cf247b 100644 --- a/rs/ic_os/network/src/systemd.rs +++ b/rs/ic_os/network/src/systemd.rs @@ -7,7 +7,7 @@ use anyhow::{Context, Result}; use crate::interfaces::{get_interfaces, has_ipv6_connectivity, Interface}; use crate::mac_address::FormattedMacAddress; -use config::types::NetworkSettings; +use config::types::DeterministicIpv6Config; pub static DEFAULT_SYSTEMD_NETWORK_DIR: &str = "/run/systemd/network"; @@ -149,7 +149,7 @@ fn generate_and_write_systemd_files( pub fn generate_systemd_config_files( output_directory: &Path, - network_settings: &NetworkSettings, + ipv6_config: &DeterministicIpv6Config, generated_mac: Option<&FormattedMacAddress>, ipv6_address: &Ipv6Addr, ) -> Result<()> { @@ -157,18 +157,13 @@ pub fn generate_systemd_config_files( interfaces.sort_by(|a, b| a.speed_mbps.cmp(&b.speed_mbps)); eprintln!("Interfaces sorted by speed: {:?}", interfaces); - let ping_target = network_settings.ipv6_gateway.to_string(); + let ping_target = ipv6_config.gateway.to_string(); // old nodes are still configured with a local IPv4 interface connection // local IPv4 interfaces must be filtered out let ipv6_interfaces: Vec<&Interface> = interfaces .iter() .filter(|i| { - match has_ipv6_connectivity( - i, - ipv6_address, - network_settings.ipv6_prefix_length, - &ping_target, - ) { + match has_ipv6_connectivity(i, ipv6_address, ipv6_config.prefix_length, &ping_target) { Ok(result) => result, Err(e) => { eprintln!("Error testing connectivity on {}: {}", &i.name, e); @@ -191,14 +186,14 @@ pub fn generate_systemd_config_files( let ipv6_address = format!( "{}/{}", &ipv6_address.to_string(), - network_settings.ipv6_prefix_length + ipv6_config.prefix_length ); generate_and_write_systemd_files( output_directory, fastest_interface, generated_mac, &ipv6_address, - &network_settings.ipv6_gateway.to_string(), + &ipv6_config.gateway.to_string(), )?; print!("Restarting systemd networkd"); diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index d072d0a524f..89177373147 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -5,7 +5,7 @@ use clap::{Parser, Subcommand}; use config::config_ini::get_config_ini_settings; use config::deployment_json::get_deployment_settings; -use config::types::NetworkSettings; +use config::types::{DeterministicIpv6Config, Ipv4Config, Ipv6Config, NetworkSettings}; use config::{DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH, DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH}; use network::generate_network_config; use network::ipv6::generate_ipv6_address; @@ -66,18 +66,26 @@ pub fn main() -> Result<()> { // Once HostOS is using the config struct, all config will be contained there // and we won't need to read config.ini and deployment.json directly. let network_settings = NetworkSettings { - ipv6_prefix: config_ini_settings.ipv6_prefix, - ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, - ipv6_gateway: config_ini_settings.ipv6_gateway, - ipv4_address: config_ini_settings.ipv4_address, - ipv4_gateway: config_ini_settings.ipv4_gateway, - ipv4_prefix_length: config_ini_settings.ipv4_prefix_length, - domain: config_ini_settings.domain, - mgmt_mac: deployment_json_settings.deployment.mgmt_mac, + ipv6_config: Ipv6Config::Deterministic(DeterministicIpv6Config { + prefix: config_ini_settings.ipv6_prefix, + prefix_length: config_ini_settings.ipv6_prefix_length, + gateway: config_ini_settings.ipv6_gateway, + }), + ipv4_config: config_ini_settings + .ipv4_address + .zip(config_ini_settings.ipv4_gateway) + .zip(config_ini_settings.ipv4_prefix_length) + .zip(config_ini_settings.domain) + .map(|(((address, gateway), prefix_length), domain)| Ipv4Config { + address, + gateway, + prefix_length, + domain, + }), }; eprintln!("Network settings config: {:?}", &network_settings); - let mgmt_mac = match network_settings.mgmt_mac.as_ref() { + let mgmt_mac = match deployment_json_settings.deployment.mgmt_mac.as_ref() { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -112,19 +120,27 @@ pub fn main() -> Result<()> { // Once HostOS is using the config struct, all config will be contained there // and we won't need to read config.ini and deployment.json directly. let network_settings = NetworkSettings { - ipv6_prefix: config_ini_settings.ipv6_prefix, - ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, - ipv6_gateway: config_ini_settings.ipv6_gateway, - ipv4_address: config_ini_settings.ipv4_address, - ipv4_gateway: config_ini_settings.ipv4_gateway, - ipv4_prefix_length: config_ini_settings.ipv4_prefix_length, - domain: config_ini_settings.domain, - mgmt_mac: deployment_json_settings.deployment.mgmt_mac, + ipv6_config: Ipv6Config::Deterministic(DeterministicIpv6Config { + prefix: config_ini_settings.ipv6_prefix, + prefix_length: config_ini_settings.ipv6_prefix_length, + gateway: config_ini_settings.ipv6_gateway, + }), + ipv4_config: config_ini_settings + .ipv4_address + .zip(config_ini_settings.ipv4_gateway) + .zip(config_ini_settings.ipv4_prefix_length) + .zip(config_ini_settings.domain) + .map(|(((address, gateway), prefix_length), domain)| Ipv4Config { + address, + gateway, + prefix_length, + domain, + }), }; eprintln!("Network settings config: {:?}", &network_settings); let node_type = node_type.parse::()?; - let mgmt_mac = match network_settings.mgmt_mac.as_ref() { + let mgmt_mac = match deployment_json_settings.deployment.mgmt_mac.as_ref() { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -141,12 +157,17 @@ pub fn main() -> Result<()> { &node_type, )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); - let ipv6_address = - generate_ipv6_address(&network_settings.ipv6_prefix, &generated_mac)?; - println!( - "{}", - to_cidr(ipv6_address, network_settings.ipv6_prefix_length) - ); + + let ipv6_config = + if let Ipv6Config::Deterministic(ipv6_config) = &network_settings.ipv6_config { + ipv6_config + } else { + return Err(anyhow!("Ipv6Config is not of type Deterministic")); + }; + + let ipv6_address = generate_ipv6_address(&ipv6_config.prefix, &generated_mac)?; + println!("{}", to_cidr(ipv6_address, ipv6_config.prefix_length)); + Ok(()) } Some(Commands::GenerateMacAddress { node_type }) => { @@ -160,19 +181,27 @@ pub fn main() -> Result<()> { // Once HostOS is using the config struct, all config will be contained there // and we won't need to read config.ini and deployment.json directly. let network_settings = NetworkSettings { - ipv6_prefix: config_ini_settings.ipv6_prefix, - ipv6_prefix_length: config_ini_settings.ipv6_prefix_length, - ipv6_gateway: config_ini_settings.ipv6_gateway, - ipv4_address: config_ini_settings.ipv4_address, - ipv4_gateway: config_ini_settings.ipv4_gateway, - ipv4_prefix_length: config_ini_settings.ipv4_prefix_length, - domain: config_ini_settings.domain, - mgmt_mac: deployment_json_settings.deployment.mgmt_mac, + ipv6_config: Ipv6Config::Deterministic(DeterministicIpv6Config { + prefix: config_ini_settings.ipv6_prefix, + prefix_length: config_ini_settings.ipv6_prefix_length, + gateway: config_ini_settings.ipv6_gateway, + }), + ipv4_config: config_ini_settings + .ipv4_address + .zip(config_ini_settings.ipv4_gateway) + .zip(config_ini_settings.ipv4_prefix_length) + .zip(config_ini_settings.domain) + .map(|(((address, gateway), prefix_length), domain)| Ipv4Config { + address, + gateway, + prefix_length, + domain, + }), }; eprintln!("Network settings config: {:?}", &network_settings); let node_type = node_type.parse::()?; - let mgmt_mac = match network_settings.mgmt_mac.as_ref() { + let mgmt_mac = match deployment_json_settings.deployment.mgmt_mac.as_ref() { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index e53d6cf220e..2178d6e5ead 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -3,7 +3,7 @@ use std::path::Path; use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; -use config::types::SetupOSConfig; +use config::types::{Ipv6Config, SetupOSConfig}; use config::{ deserialize_config, DEFAULT_SETUPOS_CONFIG_INI_FILE_PATH, DEFAULT_SETUPOS_CONFIG_OBJECT_PATH, DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH, @@ -60,7 +60,12 @@ pub fn main() -> Result<()> { &setup_config.network_settings ); - let mgmt_mac = match setup_config.network_settings.mgmt_mac.as_ref() { + let mgmt_mac = match setup_config + .icos_settings + .icos_dev_settings + .mgmt_mac + .as_ref() + { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -94,7 +99,7 @@ pub fn main() -> Result<()> { ); let node_type = node_type.parse::()?; - let mgmt_mac = match setup_config.network_settings.mgmt_mac { + let mgmt_mac = match setup_config.icos_settings.icos_dev_settings.mgmt_mac { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -109,15 +114,16 @@ pub fn main() -> Result<()> { generate_mac_address(&mgmt_mac, &setup_config.icos_settings.hostname, &node_type)?; eprintln!("Using generated mac (unformatted) {}", generated_mac); - let ipv6_address = - generate_ipv6_address(&setup_config.network_settings.ipv6_prefix, &generated_mac)?; - println!( - "{}", - to_cidr( - ipv6_address, - setup_config.network_settings.ipv6_prefix_length - ) - ); + let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = + &setup_config.network_settings.ipv6_config + { + ipv6_config + } else { + return Err(anyhow!("Ipv6Config is not of type Deterministic")); + }; + + let ipv6_address = generate_ipv6_address(&ipv6_config.prefix, &generated_mac)?; + println!("{}", to_cidr(ipv6_address, ipv6_config.prefix_length)); Ok(()) } From ef606d07576a97d19c5b3495cd6b9224cafa962e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 19:26:41 +0000 Subject: [PATCH 016/241] Copy config-hostos.json to HostOS --- .../setupos-scripts/setup-hostos-config.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 181eb119487..d1308c49495 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -66,14 +66,13 @@ function copy_config_files() { /opt/ic/bin/config generate-hostos-config log_and_halt_installation_on_error "${?}" "Unable to generate hostos configuration." - # TODO: NODE-1466: Configuration revamp (HostOS and GuestOS integration) - # echo "* Copying 'config-hostos.json' to hostOS config partition..." - # if [ -f "/var/ic/config/config-hostos.json" ]; then - # cp /var/ic/config/config-hostos.json /media/config.json - # log_and_halt_installation_on_error "${?}" "Unable to copy 'config-hostos.json' to hostOS config partition." - # else - # log_and_halt_installation_on_error "1" "Configuration file 'config-hostos.json' does not exist." - # fi + echo "* Copying 'config-hostos.json' to hostOS config partition..." + if [ -f "/var/ic/config/config-hostos.json" ]; then + cp /var/ic/config/config-hostos.json /media/config.json + log_and_halt_installation_on_error "${?}" "Unable to copy 'config-hostos.json' to hostOS config partition." + else + log_and_halt_installation_on_error "1" "Configuration file 'config-hostos.json' does not exist." + fi } function insert_hsm_if_necessary() { From e08ab58d0caca3ade68b4b5dc7542f08ba7769f0 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 19:26:55 +0000 Subject: [PATCH 017/241] Update GenerateIpv6Address error message --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 4 +++- rs/ic_os/os_tools/setupos_tool/src/main.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 89177373147..3362ae50051 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -162,7 +162,9 @@ pub fn main() -> Result<()> { if let Ipv6Config::Deterministic(ipv6_config) = &network_settings.ipv6_config { ipv6_config } else { - return Err(anyhow!("Ipv6Config is not of type Deterministic")); + return Err(anyhow!( + "Ipv6Config is not of type Deterministic. Cannot generate IPv6 address." + )); }; let ipv6_address = generate_ipv6_address(&ipv6_config.prefix, &generated_mac)?; diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 2178d6e5ead..bd809ee697a 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -119,7 +119,9 @@ pub fn main() -> Result<()> { { ipv6_config } else { - return Err(anyhow!("Ipv6Config is not of type Deterministic")); + return Err(anyhow!( + "Ipv6Config is not of type Deterministic. Cannot generate IPv6 address." + )); }; let ipv6_address = generate_ipv6_address(&ipv6_config.prefix, &generated_mac)?; From 5f262b5b2d5a21fc7388900cdfc7c2954a922471 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 19:27:08 +0000 Subject: [PATCH 018/241] Add log_start and log_end to check-config.sh --- ic-os/components/setupos-scripts/check-config.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ic-os/components/setupos-scripts/check-config.sh b/ic-os/components/setupos-scripts/check-config.sh index b826345a4c5..fa20310e9fc 100644 --- a/ic-os/components/setupos-scripts/check-config.sh +++ b/ic-os/components/setupos-scripts/check-config.sh @@ -27,7 +27,9 @@ check_config_file() { # Establish run order main() { + log_start "$(basename $0)" check_config_file + log_end "$(basename $0)" } main From 664ed5bfd22699e73a1f217a82ba85ff10cfe7de Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 19:29:13 +0000 Subject: [PATCH 019/241] Rename setupos_config --- rs/ic_os/os_tools/setupos_tool/src/main.rs | 25 ++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index bd809ee697a..152c6c48e24 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -52,15 +52,15 @@ pub fn main() -> Result<()> { match opts.command { Some(Commands::GenerateNetworkConfig { output_directory }) => { - let setup_config: SetupOSConfig = + let setupos_config: SetupOSConfig = deserialize_config(DEFAULT_SETUPOS_CONFIG_OBJECT_PATH)?; eprintln!( "Network settings config: {:?}", - &setup_config.network_settings + &setupos_config.network_settings ); - let mgmt_mac = match setup_config + let mgmt_mac = match setupos_config .icos_settings .icos_dev_settings .mgmt_mac @@ -78,28 +78,28 @@ pub fn main() -> Result<()> { }; let generated_mac = generate_mac_address( &mgmt_mac, - &setup_config.icos_settings.hostname, + &setupos_config.icos_settings.hostname, &NodeType::SetupOS, )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); generate_network_config( - &setup_config.network_settings, + &setupos_config.network_settings, generated_mac, Path::new(&output_directory), ) } Some(Commands::GenerateIpv6Address { node_type }) => { - let setup_config: SetupOSConfig = + let setupos_config: SetupOSConfig = deserialize_config(DEFAULT_SETUPOS_CONFIG_OBJECT_PATH)?; eprintln!( "Network settings config: {:?}", - &setup_config.network_settings + &setupos_config.network_settings ); let node_type = node_type.parse::()?; - let mgmt_mac = match setup_config.icos_settings.icos_dev_settings.mgmt_mac { + let mgmt_mac = match setupos_config.icos_settings.icos_dev_settings.mgmt_mac { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -110,12 +110,15 @@ pub fn main() -> Result<()> { } None => get_ipmi_mac()?, }; - let generated_mac = - generate_mac_address(&mgmt_mac, &setup_config.icos_settings.hostname, &node_type)?; + let generated_mac = generate_mac_address( + &mgmt_mac, + &setupos_config.icos_settings.hostname, + &node_type, + )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = - &setup_config.network_settings.ipv6_config + &setupos_config.network_settings.ipv6_config { ipv6_config } else { From 328305eabfbbf9d49f3eb2615976003e58026479 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 19:40:13 +0000 Subject: [PATCH 020/241] Use hostos config object in HostOS tool --- rs/ic_os/config/src/lib.rs | 1 + rs/ic_os/os_tools/hostos_tool/src/main.rs | 184 ++++++++-------------- 2 files changed, 69 insertions(+), 116 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index d10ef35b8e4..1b80cd41bc9 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -18,6 +18,7 @@ pub static DEFAULT_SETUPOS_NODE_OPERATOR_PRIVATE_KEY_PATH: &str = pub static DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/config-hostos.json"; +pub static DEFAULT_HOSTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config-hostos.json"; pub static DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH: &str = "/boot/config/config.ini"; pub static DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH: &str = "/boot/config/deployment.json"; diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 3362ae50051..dd2ea5c60f6 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -3,10 +3,11 @@ use std::path::Path; use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; -use config::config_ini::get_config_ini_settings; -use config::deployment_json::get_deployment_settings; -use config::types::{DeterministicIpv6Config, Ipv4Config, Ipv6Config, NetworkSettings}; -use config::{DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH, DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH}; +use config::types::{HostOSConfig, Ipv6Config}; +use config::{ + deserialize_config, DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH, DEFAULT_HOSTOS_CONFIG_OBJECT_PATH, + DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH, +}; use network::generate_network_config; use network::ipv6::generate_ipv6_address; use network::mac_address::{generate_mac_address, get_ipmi_mac, FormattedMacAddress}; @@ -56,36 +57,20 @@ pub fn main() -> Result<()> { match opts.command { Some(Commands::GenerateNetworkConfig { output_directory }) => { - let config_ini_settings = get_config_ini_settings(Path::new(&opts.config))?; - - let deployment_json_settings = - get_deployment_settings(Path::new(&opts.deployment_file))?; - eprintln!("Deployment config: {:?}", deployment_json_settings); - - // TODO: NODE-1466: Remove in configuration revamp (HostOS and GuestOS integration). - // Once HostOS is using the config struct, all config will be contained there - // and we won't need to read config.ini and deployment.json directly. - let network_settings = NetworkSettings { - ipv6_config: Ipv6Config::Deterministic(DeterministicIpv6Config { - prefix: config_ini_settings.ipv6_prefix, - prefix_length: config_ini_settings.ipv6_prefix_length, - gateway: config_ini_settings.ipv6_gateway, - }), - ipv4_config: config_ini_settings - .ipv4_address - .zip(config_ini_settings.ipv4_gateway) - .zip(config_ini_settings.ipv4_prefix_length) - .zip(config_ini_settings.domain) - .map(|(((address, gateway), prefix_length), domain)| Ipv4Config { - address, - gateway, - prefix_length, - domain, - }), - }; - eprintln!("Network settings config: {:?}", &network_settings); - - let mgmt_mac = match deployment_json_settings.deployment.mgmt_mac.as_ref() { + let hostos_config: HostOSConfig = + deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + + eprintln!( + "Network settings config: {:?}", + &hostos_config.network_settings + ); + + let mgmt_mac = match hostos_config + .icos_settings + .icos_dev_settings + .mgmt_mac + .as_ref() + { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -96,51 +81,35 @@ pub fn main() -> Result<()> { } None => get_ipmi_mac()?, }; + let generated_mac = generate_mac_address( &mgmt_mac, - deployment_json_settings.deployment.name.as_str(), + &hostos_config.icos_settings.hostname, &NodeType::HostOS, )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); generate_network_config( - &network_settings, + &hostos_config.network_settings, generated_mac, Path::new(&output_directory), ) } Some(Commands::GenerateIpv6Address { node_type }) => { - let config_ini_settings = get_config_ini_settings(Path::new(&opts.config))?; - - let deployment_json_settings = - get_deployment_settings(Path::new(&opts.deployment_file))?; - eprintln!("Deployment config: {:?}", deployment_json_settings); - - // TODO: NODE-1466: Remove in configuration revamp (HostOS and GuestOS integration). - // Once HostOS is using the config struct, all config will be contained there - // and we won't need to read config.ini and deployment.json directly. - let network_settings = NetworkSettings { - ipv6_config: Ipv6Config::Deterministic(DeterministicIpv6Config { - prefix: config_ini_settings.ipv6_prefix, - prefix_length: config_ini_settings.ipv6_prefix_length, - gateway: config_ini_settings.ipv6_gateway, - }), - ipv4_config: config_ini_settings - .ipv4_address - .zip(config_ini_settings.ipv4_gateway) - .zip(config_ini_settings.ipv4_prefix_length) - .zip(config_ini_settings.domain) - .map(|(((address, gateway), prefix_length), domain)| Ipv4Config { - address, - gateway, - prefix_length, - domain, - }), - }; - eprintln!("Network settings config: {:?}", &network_settings); - - let node_type = node_type.parse::()?; - let mgmt_mac = match deployment_json_settings.deployment.mgmt_mac.as_ref() { + let hostos_config: HostOSConfig = + deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + + eprintln!( + "Network settings config: {:?}", + &hostos_config.network_settings + ); + + let mgmt_mac = match hostos_config + .icos_settings + .icos_dev_settings + .mgmt_mac + .as_ref() + { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -151,21 +120,21 @@ pub fn main() -> Result<()> { } None => get_ipmi_mac()?, }; - let generated_mac = generate_mac_address( - &mgmt_mac, - deployment_json_settings.deployment.name.as_str(), - &node_type, - )?; + let node_type = node_type.parse::()?; + + let generated_mac = + generate_mac_address(&mgmt_mac, &hostos_config.icos_settings.hostname, &node_type)?; eprintln!("Using generated mac (unformatted) {}", generated_mac); - let ipv6_config = - if let Ipv6Config::Deterministic(ipv6_config) = &network_settings.ipv6_config { - ipv6_config - } else { - return Err(anyhow!( - "Ipv6Config is not of type Deterministic. Cannot generate IPv6 address." - )); - }; + let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = + &hostos_config.network_settings.ipv6_config + { + ipv6_config + } else { + return Err(anyhow!( + "Ipv6Config is not of type Deterministic. Cannot generate IPv6 address." + )); + }; let ipv6_address = generate_ipv6_address(&ipv6_config.prefix, &generated_mac)?; println!("{}", to_cidr(ipv6_address, ipv6_config.prefix_length)); @@ -173,37 +142,20 @@ pub fn main() -> Result<()> { Ok(()) } Some(Commands::GenerateMacAddress { node_type }) => { - let config_ini_settings = get_config_ini_settings(Path::new(&opts.config))?; - - let deployment_json_settings = - get_deployment_settings(Path::new(&opts.deployment_file))?; - eprintln!("Deployment config: {:?}", deployment_json_settings); - - // TODO: NODE-1466: Remove in configuration revamp (HostOS and GuestOS integration). - // Once HostOS is using the config struct, all config will be contained there - // and we won't need to read config.ini and deployment.json directly. - let network_settings = NetworkSettings { - ipv6_config: Ipv6Config::Deterministic(DeterministicIpv6Config { - prefix: config_ini_settings.ipv6_prefix, - prefix_length: config_ini_settings.ipv6_prefix_length, - gateway: config_ini_settings.ipv6_gateway, - }), - ipv4_config: config_ini_settings - .ipv4_address - .zip(config_ini_settings.ipv4_gateway) - .zip(config_ini_settings.ipv4_prefix_length) - .zip(config_ini_settings.domain) - .map(|(((address, gateway), prefix_length), domain)| Ipv4Config { - address, - gateway, - prefix_length, - domain, - }), - }; - eprintln!("Network settings config: {:?}", &network_settings); - - let node_type = node_type.parse::()?; - let mgmt_mac = match deployment_json_settings.deployment.mgmt_mac.as_ref() { + let hostos_config: HostOSConfig = + deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + + eprintln!( + "Network settings config: {:?}", + &hostos_config.network_settings + ); + + let mgmt_mac = match hostos_config + .icos_settings + .icos_dev_settings + .mgmt_mac + .as_ref() + { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( @@ -214,11 +166,11 @@ pub fn main() -> Result<()> { } None => get_ipmi_mac()?, }; - let generated_mac = generate_mac_address( - &mgmt_mac, - deployment_json_settings.deployment.name.as_str(), - &node_type, - )?; + let node_type = node_type.parse::()?; + + let generated_mac = + generate_mac_address(&mgmt_mac, &hostos_config.icos_settings.hostname, &node_type)?; + eprintln!("Using generated mac (unformatted) {}", generated_mac); println!("{}", generated_mac); Ok(()) From 83ab1d092bcd9e18d2bb42425f2928ad7160f443 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 20:44:55 +0000 Subject: [PATCH 021/241] Create separate config.sh for hostos and setupos --- ic-os/components/hostos.bzl | 1 + ic-os/components/misc/config/hostos-config.sh | 12 ++++++++++++ .../{config.sh => config/setupos-config copy.sh} | 0 ic-os/components/setupos.bzl | 2 +- 4 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 ic-os/components/misc/config/hostos-config.sh rename ic-os/components/misc/{config.sh => config/setupos-config copy.sh} (100%) diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index c40901fbcaa..7d074a4d5f4 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -48,6 +48,7 @@ component_files = { Label("early-boot/initramfs-tools/hostos/set-machine-id/set-machine-id"): "/etc/initramfs-tools/scripts/init-bottom/set-machine-id/set-machine-id", # misc + Label("misc/config/hostos-config.sh"): "/opt/ic/bin/config.sh", Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", Label("misc/metrics.sh"): "/opt/ic/bin/metrics.sh", Label("misc/fetch-property.sh"): "/opt/ic/bin/fetch-property.sh", diff --git a/ic-os/components/misc/config/hostos-config.sh b/ic-os/components/misc/config/hostos-config.sh new file mode 100644 index 00000000000..831d04cbc28 --- /dev/null +++ b/ic-os/components/misc/config/hostos-config.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Shared config utilities. + +# Retrieves a value from the config.json file using a JSON path. +# Arguments: +# $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') +function get_config_value() { + local CONFIG_FILE="/boot/config/config.json" + local key=$1 + jq -r "${key}" "${CONFIG_FILE}" +} diff --git a/ic-os/components/misc/config.sh b/ic-os/components/misc/config/setupos-config copy.sh similarity index 100% rename from ic-os/components/misc/config.sh rename to ic-os/components/misc/config/setupos-config copy.sh diff --git a/ic-os/components/setupos.bzl b/ic-os/components/setupos.bzl index c55ea1715b8..1c2afa576b3 100644 --- a/ic-os/components/setupos.bzl +++ b/ic-os/components/setupos.bzl @@ -26,7 +26,7 @@ component_files = { # misc Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", - Label("misc/config.sh"): "/opt/ic/bin/config.sh", + Label("misc/config/setupos-config.sh"): "/opt/ic/bin/config.sh", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", Label("misc/chrony/chrony-var.service"): "/etc/systemd/system/chrony-var.service", Label("misc/serial-getty@/setupos/serial-getty@.service"): "/etc/systemd/system/serial-getty@.service", From f915d288b62381aa90eb2de50de980b126cfc515 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 20:45:14 +0000 Subject: [PATCH 022/241] Fix reference to DEFAULT_HOSTOS_CONFIG_OBJECT_PATH --- rs/ic_os/config/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 1b80cd41bc9..40014d8e590 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -18,7 +18,7 @@ pub static DEFAULT_SETUPOS_NODE_OPERATOR_PRIVATE_KEY_PATH: &str = pub static DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/config-hostos.json"; -pub static DEFAULT_HOSTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config-hostos.json"; +pub static DEFAULT_HOSTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config.json"; pub static DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH: &str = "/boot/config/config.ini"; pub static DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH: &str = "/boot/config/deployment.json"; From c098cf47e228a4f173516b990ffce260f7e5b68e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 20:45:35 +0000 Subject: [PATCH 023/241] Update verbose-logging to use config object --- .../verbose-logging/verbose-logging.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh b/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh index a8ff2e9b2cd..979464d0777 100644 --- a/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh +++ b/ic-os/components/hostos-scripts/verbose-logging/verbose-logging.sh @@ -1,18 +1,8 @@ #!/bin/bash -CONFIG="${CONFIG:=/boot/config/config.ini}" +source /opt/ic/bin/config.sh -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "verbose") verbose="${value}" ;; - esac - done <"${CONFIG}" -} - -read_variables +verbose=$(get_config_value '.hostos_settings.verbose') if [[ "${verbose,,}" == "true" ]]; then echo "##########################################" >/dev/tty1 From 47d9b4efc7bdcf21563f79956511d288ae99c265 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 20:46:22 +0000 Subject: [PATCH 024/241] Update generate-guestos-config to use config object and delete unused fetch-mgmt-mac.sh --- .../check_file_references.py | 7 +- .../build-bootstrap-config-image.sh | 12 +-- .../dev-generate-guestos-config.sh | 75 +++++--------- .../generate-guestos-config.sh | 63 +++++------- .../hostos-scripts/misc/fetch-mgmt-mac.sh | 2 - ic-os/components/hostos.bzl | 1 - ic-os/components/misc/fetch-property.sh | 98 ------------------- 7 files changed, 56 insertions(+), 202 deletions(-) delete mode 100755 ic-os/components/misc/fetch-property.sh diff --git a/ic-os/components/conformance_tests/check_file_references.py b/ic-os/components/conformance_tests/check_file_references.py index cd91b8723e7..6009ff44384 100755 --- a/ic-os/components/conformance_tests/check_file_references.py +++ b/ic-os/components/conformance_tests/check_file_references.py @@ -11,12 +11,7 @@ import tarfile import tempfile -ALLOWED_UNDECLARED_DEPENDENCIES = { - "ic-os/components/misc/fetch-property.sh": { - # fetch-property.sh checks existence of metrics.sh - "/opt/ic/bin/metrics.sh", - } -} +ALLOWED_UNDECLARED_DEPENDENCIES = {} # Check file patterns /opt/ic/... COMPONENT_FILE_PATTERN = r"/opt/ic/[^\s'\"},)]+" diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index ec1f21cfd1e..2f759d4e199 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -63,7 +63,7 @@ options may be specified: (make sure to quote the argument string so it appears as a single argument to the script, e.g. --elasticsearch_tags "testnet1 slo") - --nns_url url + --nns_urls urls URL of NNS nodes for sign up or registry access. Can be multiple nodes separated by commas. @@ -122,7 +122,7 @@ function build_ic_bootstrap_tar() { local IPV6_ADDRESS IPV6_GATEWAY DOMAIN HOSTNAME local IC_CRYPTO IC_STATE IC_REGISTRY_LOCAL_STORE - local NNS_URL NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY + local NNS_URLS NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY local BACKUP_RETENTION_TIME_SECS BACKUP_PURGING_INTERVAL_SECS local ELASTICSEARCH_HOSTS ELASTICSEARCH_TAGS local ACCOUNTS_SSH_AUTHORIZED_KEYS @@ -170,8 +170,8 @@ function build_ic_bootstrap_tar() { --elasticsearch_tags) ELASTICSEARCH_TAGS="$2" ;; - --nns_url) - NNS_URL="$2" + --nns_urls) + NNS_URLS="$2" ;; --nns_public_key) NNS_PUBLIC_KEY="$2" @@ -237,8 +237,8 @@ EOF if [ "${NNS_PUBLIC_KEY}" != "" ]; then cp "${NNS_PUBLIC_KEY}" "${BOOTSTRAP_TMPDIR}/nns_public_key.pem" fi - if [ "${NNS_URL}" != "" ]; then - echo "nns_url=${NNS_URL}" >"${BOOTSTRAP_TMPDIR}/nns.conf" + if [ "${NNS_URLS}" != "" ]; then + echo "nns_url=${NNS_URLS}" >"${BOOTSTRAP_TMPDIR}/nns.conf" fi if [ "${BACKUP_RETENTION_TIME_SECS}" != "" ] || [ "${BACKUP_PURGING_INTERVAL_SECS}" != "" ]; then echo "backup_retention_time_secs=${BACKUP_RETENTION_TIME_SECS}" >"${BOOTSTRAP_TMPDIR}/backup.conf" diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 49276f8719b..c1a2f809cb0 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -4,6 +4,7 @@ set -e # Generate the GuestOS configuration. +source /opt/ic/bin/logging.sh # Source the functions required for writing metrics source /opt/ic/bin/metrics.sh @@ -12,21 +13,11 @@ SCRIPT="$(basename $0)[$$]" # Get keyword arguments for argument in "${@}"; do case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; - -d=* | --deployment=*) - DEPLOYMENT="${argument#*=}" - shift - ;; -h | --help) echo 'Usage: Generate GuestOS Configuration Arguments: - -c=, --config= specify the config.ini configuration file (Default: /boot/config/config.ini) - -d=, --deployment= specify the deployment.json configuration file (Default: /boot/config/deployment.json) -h, --help show this help message and exit -i=, --input= specify the input template file (Default: /opt/ic/share/guestos.xml.template) -m=, --media= specify the config media image file (Default: /run/ic-node/config.img) @@ -54,47 +45,35 @@ Arguments: done function validate_arguments() { - if [ "${CONFIG}" == "" -o "${DEPLOYMENT}" == "" -o "${INPUT}" == "" -o "${OUTPUT}" == "" ]; then + if [ "${INPUT}" == "" -o "${OUTPUT}" == "" ]; then $0 --help fi } # Set arguments if undefined -CONFIG="${CONFIG:=/boot/config/config.ini}" -DEPLOYMENT="${DEPLOYMENT:=/boot/config/deployment.json}" INPUT="${INPUT:=/opt/ic/share/guestos.xml.template}" MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" -write_log() { - local message=$1 - - if [ -t 1 ]; then - echo "${SCRIPT} ${message}" >/dev/stdout - fi - - logger -t ${SCRIPT} "${message}" -} - -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; - "ipv4_address") ipv4_address="${value}" ;; - "ipv4_prefix_length") ipv4_prefix_length="${value}" ;; - "ipv4_gateway") ipv4_gateway="${value}" ;; - "domain") domain="${value}" ;; - esac - done <"${CONFIG}" +function read_config_variables() { + ipv6_prefix=$(get_config_value '.network_settings.ipv6_prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_gateway') + domain=$(get_config_value '.network_settings.domain') + elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') + nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') + nns_urls=$(get_config_value '.icos_settings.nns_urls') + node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') + vm_memory=$(get_config_value '.hostos_settings.vm_memory') + vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') } function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) - cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") - cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") + cmd+=(--nns_public_key "$nns_public_key") + cmd+=(--elasticsearch_hosts "$elasticsearch_hosts") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then @@ -102,12 +81,12 @@ function assemble_config_media() { cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain}") fi + # todo: can I use the fetch-mgmt-mac in hostos tool? cmd+=(--hostname "guest-$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g')") - cmd+=(--nns_url "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") - if [ -f "/boot/config/node_operator_private_key.pem" ]; then - cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") + cmd+=(--nns_urls "$nns_urls") + if [ -f "$node_operator_private_key" ]; then + cmd+=(--node_operator_private_key "$node_operator_private_key") fi - cmd+=(--accounts_ssh_authorized_keys "/boot/config/ssh_authorized_keys") # Run the above command @@ -116,22 +95,19 @@ function assemble_config_media() { } function generate_guestos_config() { - RESOURCES_MEMORY=$(/opt/ic/bin/fetch-property.sh --key=.resources.memory --metric=hostos_resources_memory --config=${DEPLOYMENT}) + # todo: can I use the generate mac address in hostos tool? MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) - # NOTE: `fetch-property` will error if the target is not found. Here we - # only want to act when the field is set. - CPU_MODE=$(jq -r ".resources.cpu" ${DEPLOYMENT}) CPU_DOMAIN="kvm" CPU_SPEC="/opt/ic/share/kvm-cpu.xml" - if [ "${CPU_MODE}" == "qemu" ]; then + if [ "${vm_cpu}" == "qemu" ]; then CPU_DOMAIN="qemu" CPU_SPEC="/opt/ic/share/qemu-cpu.xml" fi if [ ! -f "${OUTPUT}" ]; then mkdir -p "$(dirname "$OUTPUT")" - sed -e "s@{{ resources_memory }}@${RESOURCES_MEMORY}@" \ + sed -e "s@{{ resources_memory }}@${vm_memory}@" \ -e "s@{{ mac_address }}@${MAC_ADDRESS}@" \ -e "s@{{ cpu_domain }}@${CPU_DOMAIN}@" \ -e "/{{ cpu_spec }}/{r ${CPU_SPEC}" -e "d" -e "}" \ @@ -152,9 +128,8 @@ function generate_guestos_config() { } function main() { - # Establish run order validate_arguments - read_variables + read_config_variables assemble_config_media generate_guestos_config } diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index cca55130bf4..439c19fc4a2 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -13,21 +13,11 @@ SCRIPT="$(basename $0)[$$]" # Get keyword arguments for argument in "${@}"; do case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; - -d=* | --deployment=*) - DEPLOYMENT="${argument#*=}" - shift - ;; -h | --help) echo 'Usage: Generate GuestOS Configuration Arguments: - -c=, --config= specify the config.ini configuration file (Default: /boot/config/config.ini) - -d=, --deployment= specify the deployment.json configuration file (Default: /boot/config/deployment.json) -h, --help show this help message and exit -i=, --input= specify the input template file (Default: /opt/ic/share/guestos.xml.template) -m=, --media= specify the config media image file (Default: /run/ic-node/config.img) @@ -55,37 +45,35 @@ Arguments: done function validate_arguments() { - if [ "${CONFIG}" == "" -o "${DEPLOYMENT}" == "" -o "${INPUT}" == "" -o "${OUTPUT}" == "" ]; then + if [ "${INPUT}" == "" -o "${OUTPUT}" == "" ]; then $0 --help fi } # Set arguments if undefined -CONFIG="${CONFIG:=/boot/config/config.ini}" -DEPLOYMENT="${DEPLOYMENT:=/boot/config/deployment.json}" INPUT="${INPUT:=/opt/ic/share/guestos.xml.template}" MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; - "ipv4_address") ipv4_address="${value}" ;; - "ipv4_prefix_length") ipv4_prefix_length="${value}" ;; - "ipv4_gateway") ipv4_gateway="${value}" ;; - "domain") domain="${value}" ;; - esac - done <"${CONFIG}" +function read_config_variables() { + ipv6_prefix=$(get_config_value '.network_settings.ipv6_prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_gateway') + domain=$(get_config_value '.network_settings.domain') + elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') + nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') + nns_urls=$(get_config_value '.icos_settings.nns_urls') + node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') + vm_memory=$(get_config_value '.hostos_settings.vm_memory') + vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') } function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) - cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") - cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") + cmd+=(--nns_public_key "$nns_public_key") + cmd+=(--elasticsearch_hosts "$elasticsearch_hosts") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then @@ -93,10 +81,11 @@ function assemble_config_media() { cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain}") fi + # todo: can I use the fetch-mgmt-mac in hostos tool? cmd+=(--hostname "guest-$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g')") - cmd+=(--nns_url "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") - if [ -f "/boot/config/node_operator_private_key.pem" ]; then - cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") + cmd+=(--nns_urls "$nns_urls") + if [ -f "$node_operator_private_key" ]; then + cmd+=(--node_operator_private_key "$node_operator_private_key") fi # Run the above command @@ -105,22 +94,19 @@ function assemble_config_media() { } function generate_guestos_config() { - RESOURCES_MEMORY=$(/opt/ic/bin/fetch-property.sh --key=.resources.memory --metric=hostos_resources_memory --config=${DEPLOYMENT}) + # todo: can I use the generate mac address in hostos tool? MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) - # NOTE: `fetch-property` will error if the target is not found. Here we - # only want to act when the field is set. - CPU_MODE=$(jq -r ".resources.cpu" ${DEPLOYMENT}) CPU_DOMAIN="kvm" CPU_SPEC="/opt/ic/share/kvm-cpu.xml" - if [ "${CPU_MODE}" == "qemu" ]; then + if [ "${vm_cpu}" == "qemu" ]; then CPU_DOMAIN="qemu" CPU_SPEC="/opt/ic/share/qemu-cpu.xml" fi if [ ! -f "${OUTPUT}" ]; then mkdir -p "$(dirname "$OUTPUT")" - sed -e "s@{{ resources_memory }}@${RESOURCES_MEMORY}@" \ + sed -e "s@{{ resources_memory }}@${vm_memory}@" \ -e "s@{{ mac_address }}@${MAC_ADDRESS}@" \ -e "s@{{ cpu_domain }}@${CPU_DOMAIN}@" \ -e "/{{ cpu_spec }}/{r ${CPU_SPEC}" -e "d" -e "}" \ @@ -141,9 +127,8 @@ function generate_guestos_config() { } function main() { - # Establish run order validate_arguments - read_variables + read_config_variables assemble_config_media generate_guestos_config } diff --git a/ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh b/ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh index c469920801a..80ee8abde7b 100755 --- a/ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh +++ b/ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh @@ -58,8 +58,6 @@ function fetch_mgmt_mac() { function main() { # Establish run order - # NOTE: `fetch-property` will error if the target is not found. Here we - # only want to act when the field is set. MGMT_MAC=$(jq -r ".deployment.mgmt_mac" ${DEPLOYMENT}) if [ -z "${MGMT_MAC}" ] || [ "${MGMT_MAC}" = "null" ]; then diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 7d074a4d5f4..424df253a0c 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -51,7 +51,6 @@ component_files = { Label("misc/config/hostos-config.sh"): "/opt/ic/bin/config.sh", Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", Label("misc/metrics.sh"): "/opt/ic/bin/metrics.sh", - Label("misc/fetch-property.sh"): "/opt/ic/bin/fetch-property.sh", Label("misc/vsock/vsock-agent.service"): "/etc/systemd/system/vsock-agent.service", Label("misc/vsock/10-vhost-vsock.rules"): "/etc/udev/rules.d/10-vhost-vsock.rules", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", diff --git a/ic-os/components/misc/fetch-property.sh b/ic-os/components/misc/fetch-property.sh deleted file mode 100755 index 46c6d2a2eeb..00000000000 --- a/ic-os/components/misc/fetch-property.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -# Fetch configuration property - -source /opt/ic/bin/logging.sh - -SCRIPT="$(basename $0)[$$]" - -# Get keyword arguments -for argument in "${@}"; do - case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; - -h | --help) - echo 'Usage: -Fetch Configuration Property - -Arguments: - -c=, --config= mandatory: specify the configuration file to read from - -h, --help show this help message and exit - -k=, --key= mandatory: specify the property key - -m=, --metric= optional: specify the metric name (required if metrics.sh exists) -' - exit 1 - ;; - -k=* | --key=*) - KEY="${argument#*=}" - shift - ;; - -m=* | --metric=*) - METRIC="${argument#*=}" - shift - ;; - *) - echo "Error: Argument is not supported." - exit 1 - ;; - esac -done - -function validate_arguments() { - if [ -z "${CONFIG}" ] || [ -z "${KEY}" ]; then - $0 --help - fi - - if [ -f "/opt/ic/bin/metrics.sh" ] && [ -z "${METRIC:-}" ]; then - echo "Error: METRIC is required when metrics.sh exists." - exit 1 - fi -} - -try_write_metric() { - local name=$1 - local value=$2 - local help=$3 - local type=$4 - - # metrics.sh is required for writing metrics - # metrics.sh only exists on HostOS and GuestOS, not SetupOS - if [ -f "/opt/ic/bin/metrics.sh" ]; then - source "/opt/ic/bin/metrics.sh" - write_metric "${name}" "${value}" "${help}" "${type}" - fi -} - -function fetch_property() { - PROPERTY=$(jq -r "$(echo ${KEY})" ${CONFIG}) - - if [ -z "${PROPERTY}" ] || [ "${PROPERTY}" == "null" ]; then - write_log "ERROR: Unable to fetch property: ${KEY}" - try_write_metric "$(echo ${METRIC:-})" \ - "1" \ - "Property: $(echo ${KEY})" \ - "gauge" - exit 1 - else - write_log "Using property: ${PROPERTY}" - try_write_metric "$(echo ${METRIC:-})" \ - "0" \ - "Property: $(echo ${KEY})" \ - "gauge" - echo "${PROPERTY}" - fi -} - -function main() { - # Establish run order - validate_arguments - fetch_property -} - -main From f327c2dec2208aab86e6f5a96d0cb0948fabf668 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 20:47:21 +0000 Subject: [PATCH 025/241] Remove unused SCRIPT variable --- .../generate-guestos-config/dev-generate-guestos-config.sh | 2 -- .../generate-guestos-config/generate-guestos-config.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index c1a2f809cb0..738f92467b0 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -8,8 +8,6 @@ source /opt/ic/bin/logging.sh # Source the functions required for writing metrics source /opt/ic/bin/metrics.sh -SCRIPT="$(basename $0)[$$]" - # Get keyword arguments for argument in "${@}"; do case ${argument} in diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 439c19fc4a2..8c48d47b518 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -8,8 +8,6 @@ source /opt/ic/bin/logging.sh # Source the functions required for writing metrics source /opt/ic/bin/metrics.sh -SCRIPT="$(basename $0)[$$]" - # Get keyword arguments for argument in "${@}"; do case ${argument} in From 379e4a2919940d06e54735e5c4f3fed32c5450d6 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 20:52:34 +0000 Subject: [PATCH 026/241] Clean up generate-guestos-config --- .../generate-guestos-config/dev-generate-guestos-config.sh | 4 ++-- .../generate-guestos-config/generate-guestos-config.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 738f92467b0..e102c4c945f 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -66,6 +66,7 @@ function read_config_variables() { node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') + ssh_authorized_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') } function assemble_config_media() { @@ -85,7 +86,7 @@ function assemble_config_media() { if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi - cmd+=(--accounts_ssh_authorized_keys "/boot/config/ssh_authorized_keys") + cmd+=(--accounts_ssh_authorized_keys "$ssh_authorized_keys") # Run the above command "${cmd[@]}" @@ -93,7 +94,6 @@ function assemble_config_media() { } function generate_guestos_config() { - # todo: can I use the generate mac address in hostos tool? MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) CPU_DOMAIN="kvm" diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 8c48d47b518..64e34dc78c5 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -92,7 +92,6 @@ function assemble_config_media() { } function generate_guestos_config() { - # todo: can I use the generate mac address in hostos tool? MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) CPU_DOMAIN="kvm" From 42ac934a98276ff91573354532c9c29d0de6f295 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 21:19:24 +0000 Subject: [PATCH 027/241] Fix setupos-config name --- .../misc/config/{setupos-config copy.sh => setupos-config.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ic-os/components/misc/config/{setupos-config copy.sh => setupos-config.sh} (100%) diff --git a/ic-os/components/misc/config/setupos-config copy.sh b/ic-os/components/misc/config/setupos-config.sh similarity index 100% rename from ic-os/components/misc/config/setupos-config copy.sh rename to ic-os/components/misc/config/setupos-config.sh From 59e327f2d3b3b0fa1aef55fa997656d663b4fd42 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 21:45:01 +0000 Subject: [PATCH 028/241] Fix networking read_config_variables --- .../dev-generate-guestos-config.sh | 12 +++++------ .../generate-guestos-config.sh | 12 +++++------ .../setupos-scripts/check-network.sh | 20 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index e102c4c945f..667ad867150 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -54,12 +54,12 @@ MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" function read_config_variables() { - ipv6_prefix=$(get_config_value '.network_settings.ipv6_prefix') - ipv6_gateway=$(get_config_value '.network_settings.ipv6_gateway') - ipv4_address=$(get_config_value '.network_settings.ipv4_address') - ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_prefix_length') - ipv4_gateway=$(get_config_value '.network_settings.ipv4_gateway') - domain=$(get_config_value '.network_settings.domain') + ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') + domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') nns_urls=$(get_config_value '.icos_settings.nns_urls') diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 64e34dc78c5..ae1613a4756 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -54,12 +54,12 @@ MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" function read_config_variables() { - ipv6_prefix=$(get_config_value '.network_settings.ipv6_prefix') - ipv6_gateway=$(get_config_value '.network_settings.ipv6_gateway') - ipv4_address=$(get_config_value '.network_settings.ipv4_address') - ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_prefix_length') - ipv4_gateway=$(get_config_value '.network_settings.ipv4_gateway') - domain=$(get_config_value '.network_settings.domain') + ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') + domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') nns_urls=$(get_config_value '.icos_settings.nns_urls') diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index 8eb9932a420..e008004cdc3 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -10,12 +10,12 @@ SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" function read_config_variables() { - ipv6_prefix=$(get_config_value '.network_settings.ipv6_prefix') - ipv6_gateway=$(get_config_value '.network_settings.ipv6_gateway') - ipv4_address=$(get_config_value '.network_settings.ipv4_address') - ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_prefix_length') - ipv4_gateway=$(get_config_value '.network_settings.ipv4_gateway') - domain=$(get_config_value '.network_settings.domain') + ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') + domain=$(get_config_value '.network_settings.ipv4_config.domain') } # WARNING: Uses 'eval' for command execution. @@ -115,20 +115,20 @@ function validate_domain_name() { IFS='.' read -ra domain_parts <<<"${domain}" if [ ${#domain_parts[@]} -lt 2 ]; then - log_and_halt_installation_on_error 1 "Domain validation error: less than two domain parts in domain" + log_and_halt_installation_on_error 1 "Domain validation error: less than two domain parts in domain: ${domain}" fi for domain_part in "${domain_parts[@]}"; do if [ -z "$domain_part" ] || [ ${#domain_part} -gt 63 ]; then - log_and_halt_installation_on_error 1 "Domain validation error: domain part length violation" + log_and_halt_installation_on_error 1 "Domain validation error: domain part length violation: ${domain_part}" fi if [[ $domain_part == -* ]] || [[ $domain_part == *- ]]; then - log_and_halt_installation_on_error 1 "Domain validation error: domain part starts or ends with a hyphen" + log_and_halt_installation_on_error 1 "Domain validation error: domain part starts or ends with a hyphen: ${domain_part}" fi if ! [[ $domain_part =~ ^[a-zA-Z0-9-]+$ ]]; then - log_and_halt_installation_on_error 1 "Domain validation error: invalid characters in domain part" + log_and_halt_installation_on_error 1 "Domain validation error: invalid characters in domain part: ${domain_part}" fi done } From e8335ebac3fd98ce05bea02a2913fe13926e43f9 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 3 Oct 2024 22:16:54 +0000 Subject: [PATCH 029/241] Fix generate-guestos-config sourcing --- .../generate-guestos-config/dev-generate-guestos-config.sh | 2 +- .../generate-guestos-config/generate-guestos-config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 667ad867150..7a3c9a87a1a 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -5,8 +5,8 @@ set -e # Generate the GuestOS configuration. source /opt/ic/bin/logging.sh -# Source the functions required for writing metrics source /opt/ic/bin/metrics.sh +source /opt/ic/bin/config.sh # Get keyword arguments for argument in "${@}"; do diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index ae1613a4756..54cf4d17532 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -5,8 +5,8 @@ set -e # Generate the GuestOS configuration. source /opt/ic/bin/logging.sh -# Source the functions required for writing metrics source /opt/ic/bin/metrics.sh +source /opt/ic/bin/config.sh # Get keyword arguments for argument in "${@}"; do From af9a7fbe5e245742ecf0566644c556b22d5a9a24 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 14:32:30 +0000 Subject: [PATCH 030/241] Fix file path bug --- .../setupos-scripts/setup-hostos-config.sh | 2 ++ rs/ic_os/config/src/main.rs | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index d1308c49495..43a80956335 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -21,6 +21,7 @@ function mount_config_partition() { } function copy_config_files() { + # todo: remove copying of config.ini: echo "* Copying 'config.ini' to hostOS config partition..." if [ -f "${CONFIG_DIR}/config.ini" ]; then cp ${CONFIG_DIR}/config.ini /media/ @@ -53,6 +54,7 @@ function copy_config_files() { echo >&2 "Warning: node_operator_private_key.pem does not exist, requiring HSM." fi + # todo: remove copying of config.ini: echo "* Copying deployment.json to config partition..." cp /data/deployment.json /media/ log_and_halt_installation_on_error "${?}" "Unable to copy deployment.json to hostOS config partition." diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 04686451ef4..24afbe1aa0f 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -166,9 +166,20 @@ pub fn main() -> Result<()> { let setupos_config: SetupOSConfig = serde_json::from_reader(File::open(setupos_config_json_path)?)?; + // update select file paths for HostOS + let mut hostos_icos_settings = setupos_config.icos_settings; + if let Some(ref mut path) = hostos_icos_settings.ssh_authorized_keys_path { + *path = PathBuf::from("/boot/config/ssh_authorized_keys"); + } + if let Some(ref mut path) = hostos_icos_settings.node_operator_private_key_path { + *path = PathBuf::from("/boot/config/node_operator_private_key.pem"); + } + hostos_icos_settings.nns_public_key_path = + PathBuf::from("/boot/config/nns_public_key.pem"); + let hostos_config = HostOSConfig { network_settings: setupos_config.network_settings, - icos_settings: setupos_config.icos_settings, + icos_settings: hostos_icos_settings, hostos_settings: setupos_config.hostos_settings, guestos_settings: setupos_config.guestos_settings, }; From 17c7926ca3726964e585366c3997a236c0ba61fc Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 14:57:31 +0000 Subject: [PATCH 031/241] Revert nns_url naming --- .../hostos-scripts/build-bootstrap-config-image.sh | 12 ++++++------ .../dev-generate-guestos-config.sh | 4 ++-- .../generate-guestos-config.sh | 4 ++-- ic-os/components/setupos-scripts/check-network.sh | 4 ++-- rs/ic_os/config/src/lib.rs | 2 +- rs/ic_os/config/src/main.rs | 2 +- rs/ic_os/config/src/types.rs | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 2f759d4e199..dff5ce4ff79 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -63,7 +63,7 @@ options may be specified: (make sure to quote the argument string so it appears as a single argument to the script, e.g. --elasticsearch_tags "testnet1 slo") - --nns_urls urls + --nns_url urls URL of NNS nodes for sign up or registry access. Can be multiple nodes separated by commas. @@ -122,7 +122,7 @@ function build_ic_bootstrap_tar() { local IPV6_ADDRESS IPV6_GATEWAY DOMAIN HOSTNAME local IC_CRYPTO IC_STATE IC_REGISTRY_LOCAL_STORE - local NNS_URLS NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY + local NNS_URL NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY local BACKUP_RETENTION_TIME_SECS BACKUP_PURGING_INTERVAL_SECS local ELASTICSEARCH_HOSTS ELASTICSEARCH_TAGS local ACCOUNTS_SSH_AUTHORIZED_KEYS @@ -170,8 +170,8 @@ function build_ic_bootstrap_tar() { --elasticsearch_tags) ELASTICSEARCH_TAGS="$2" ;; - --nns_urls) - NNS_URLS="$2" + --nns_url) + NNS_URL="$2" ;; --nns_public_key) NNS_PUBLIC_KEY="$2" @@ -237,8 +237,8 @@ EOF if [ "${NNS_PUBLIC_KEY}" != "" ]; then cp "${NNS_PUBLIC_KEY}" "${BOOTSTRAP_TMPDIR}/nns_public_key.pem" fi - if [ "${NNS_URLS}" != "" ]; then - echo "nns_url=${NNS_URLS}" >"${BOOTSTRAP_TMPDIR}/nns.conf" + if [ "${NNS_URL}" != "" ]; then + echo "nns_url=${NNS_URL}" >"${BOOTSTRAP_TMPDIR}/nns.conf" fi if [ "${BACKUP_RETENTION_TIME_SECS}" != "" ] || [ "${BACKUP_PURGING_INTERVAL_SECS}" != "" ]; then echo "backup_retention_time_secs=${BACKUP_RETENTION_TIME_SECS}" >"${BOOTSTRAP_TMPDIR}/backup.conf" diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 7a3c9a87a1a..7f85a99f5f3 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -62,7 +62,7 @@ function read_config_variables() { domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') - nns_urls=$(get_config_value '.icos_settings.nns_urls') + nns_url=$(get_config_value '.icos_settings.nns_url') node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') @@ -82,7 +82,7 @@ function assemble_config_media() { fi # todo: can I use the fetch-mgmt-mac in hostos tool? cmd+=(--hostname "guest-$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g')") - cmd+=(--nns_urls "$nns_urls") + cmd+=(--nns_url "$nns_url") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 54cf4d17532..a84c0e8a51e 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -62,7 +62,7 @@ function read_config_variables() { domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') - nns_urls=$(get_config_value '.icos_settings.nns_urls') + nns_url=$(get_config_value '.icos_settings.nns_url') node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') @@ -81,7 +81,7 @@ function assemble_config_media() { fi # todo: can I use the fetch-mgmt-mac in hostos tool? cmd+=(--hostname "guest-$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g')") - cmd+=(--nns_urls "$nns_urls") + cmd+=(--nns_url "$nns_url") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index e008004cdc3..09049eaa2d8 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -165,10 +165,10 @@ function ping_ipv6_gateway() { function query_nns_nodes() { echo "* Querying NNS nodes..." - local nns_urls=($(get_config_value '.icos_settings.nns_urls' | jq -r '.[]')) + local nns_url=($(get_config_value '.icos_settings.nns_url' | jq -r '.[]')) local success=false - for url in "${nns_urls[@]}"; do + for url in "${nns_url[@]}"; do # When running against testnets, we need to ignore self signed certs # with `--insecure`. This check is only meant to confirm from SetupOS # that NNS urls are reachable, so we do not mind that it is "weak". diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 40014d8e590..9a513e4161f 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -74,7 +74,7 @@ mod tests { let icos_settings = ICOSSettings { logging, nns_public_key_path: PathBuf::from("/path/to/key"), - nns_urls: vec!["http://localhost".parse().unwrap()], + nns_url: vec!["http://localhost".parse().unwrap()], hostname: "mainnet".to_string(), node_operator_private_key_path: None, ssh_authorized_keys_path: None, diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 24afbe1aa0f..8da7dd2ccaf 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -113,7 +113,7 @@ pub fn main() -> Result<()> { let icos_settings = ICOSSettings { logging, nns_public_key_path: nns_public_key_path.to_path_buf(), - nns_urls: deployment_json_settings.nns.url.clone(), + nns_url: deployment_json_settings.nns.url.clone(), hostname: deployment_json_settings.deployment.name.to_string(), node_operator_private_key_path: node_operator_private_key_path .exists() diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 1ca9963492a..a9d62c36908 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -87,7 +87,7 @@ pub struct ICOSSettings { /// This file must be a text file containing the public key of the NNS to be used. pub nns_public_key_path: PathBuf, /// The URL (HTTP) of the NNS node(s). - pub nns_urls: Vec, + pub nns_url: Vec, pub hostname: String, /// This file contains the Node Operator private key, /// which is registered with the NNS and used to sign the IC join request. From 08aa0a47e6014cf7f2b89f5acd993207bc29faa6 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 14:59:01 +0000 Subject: [PATCH 032/241] Revert additional urls reference --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index dff5ce4ff79..ec1f21cfd1e 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -63,7 +63,7 @@ options may be specified: (make sure to quote the argument string so it appears as a single argument to the script, e.g. --elasticsearch_tags "testnet1 slo") - --nns_url urls + --nns_url url URL of NNS nodes for sign up or registry access. Can be multiple nodes separated by commas. From 7f18d53b76b272f570eb84c488ca1430708df495 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 15:39:43 +0000 Subject: [PATCH 033/241] Fix bug in GenerateMacAddress --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index dd2ea5c60f6..5f1f134d956 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -172,6 +172,8 @@ pub fn main() -> Result<()> { generate_mac_address(&mgmt_mac, &hostos_config.icos_settings.hostname, &node_type)?; eprintln!("Using generated mac (unformatted) {}", generated_mac); + let generated_mac = FormattedMacAddress::from(&generated_mac); + println!("{}", generated_mac); Ok(()) } From 0d1149871838c1f703e78b8774a61153139e2e53 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 17:43:03 +0000 Subject: [PATCH 034/241] Fix nns_url jq call --- .../generate-guestos-config/dev-generate-guestos-config.sh | 2 +- .../generate-guestos-config/generate-guestos-config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 7f85a99f5f3..998afe14bcf 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -62,7 +62,7 @@ function read_config_variables() { domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') - nns_url=$(get_config_value '.icos_settings.nns_url') + nns_url=$(get_config_value '.icos_settings.nns_url | join(",")') node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index a84c0e8a51e..bc7f783c691 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -62,7 +62,7 @@ function read_config_variables() { domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') - nns_url=$(get_config_value '.icos_settings.nns_url') + nns_url=$(get_config_value '.icos_settings.nns_url | join(",")') node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') From 58bedec1857089093ad4bf45f09a6dcd7fefcc53 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 17:43:18 +0000 Subject: [PATCH 035/241] Add comment for get_ipmi_mac error log --- rs/ic_os/network/src/mac_address.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/ic_os/network/src/mac_address.rs b/rs/ic_os/network/src/mac_address.rs index 37919012bb6..ef5a6bbb078 100644 --- a/rs/ic_os/network/src/mac_address.rs +++ b/rs/ic_os/network/src/mac_address.rs @@ -124,10 +124,14 @@ pub fn generate_mac_address( /// Retrieves the MAC address from the IPMI LAN interface pub fn get_ipmi_mac() -> Result { let output = Command::new("ipmitool").arg("lan").arg("print").output()?; + + // A bug in our version of ipmitool causes it to exit with an error + // status, but we have enough output to work with anyway. + // https://github.com/ipmitool/ipmitool/issues/388 if !output.status.success() { eprintln!( "Error running ipmitool: {}", - std::str::from_utf8(&output.stderr)? + String::from_utf8_lossy(&output.stderr) ); } let ipmitool_output = String::from_utf8(output.stdout)?; From fba751c013ddf078480003a06ffed668a0d6ea68 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 17:45:39 +0000 Subject: [PATCH 036/241] Revert change to ipmitool error parsing --- rs/ic_os/network/src/mac_address.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ic_os/network/src/mac_address.rs b/rs/ic_os/network/src/mac_address.rs index ef5a6bbb078..5c632871127 100644 --- a/rs/ic_os/network/src/mac_address.rs +++ b/rs/ic_os/network/src/mac_address.rs @@ -131,7 +131,7 @@ pub fn get_ipmi_mac() -> Result { if !output.status.success() { eprintln!( "Error running ipmitool: {}", - String::from_utf8_lossy(&output.stderr) + std::str::from_utf8(&output.stderr)? ); } let ipmitool_output = String::from_utf8(output.stdout)?; From 575f73fa6eb8dd890d8f0da98181b3f9912ae05b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 19:23:04 +0000 Subject: [PATCH 037/241] Add hostos_config_path variable --- rs/ic_os/config/src/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 8da7dd2ccaf..c6be7a4fa8b 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -168,14 +168,15 @@ pub fn main() -> Result<()> { // update select file paths for HostOS let mut hostos_icos_settings = setupos_config.icos_settings; + let hostos_config_path = Path::new("/boot/config"); if let Some(ref mut path) = hostos_icos_settings.ssh_authorized_keys_path { - *path = PathBuf::from("/boot/config/ssh_authorized_keys"); + *path = hostos_config_path.join("ssh_authorized_keys"); } if let Some(ref mut path) = hostos_icos_settings.node_operator_private_key_path { - *path = PathBuf::from("/boot/config/node_operator_private_key.pem"); + *path = hostos_config_path.join("node_operator_private_key.pem"); } hostos_icos_settings.nns_public_key_path = - PathBuf::from("/boot/config/nns_public_key.pem"); + hostos_config_path.join("nns_public_key.pem"); let hostos_config = HostOSConfig { network_settings: setupos_config.network_settings, From cab3719f7a658c471cafa0f4b6da5261edda62a9 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 22:11:52 +0000 Subject: [PATCH 038/241] Remove extra function.sh calls from merge --- ic-os/components/setupos-scripts/check-hardware.sh | 2 -- ic-os/components/setupos-scripts/install-guestos.sh | 2 -- ic-os/components/setupos-scripts/install-hostos.sh | 2 -- ic-os/components/setupos-scripts/setup-disk.sh | 2 -- ic-os/components/setupos-scripts/setup-hostos-config.sh | 2 -- ic-os/components/setupos-scripts/setupos.sh | 2 -- 6 files changed, 12 deletions(-) diff --git a/ic-os/components/setupos-scripts/check-hardware.sh b/ic-os/components/setupos-scripts/check-hardware.sh index 959ba921ff0..3d77b7ac2cb 100644 --- a/ic-os/components/setupos-scripts/check-hardware.sh +++ b/ic-os/components/setupos-scripts/check-hardware.sh @@ -9,8 +9,6 @@ source /opt/ic/bin/functions.sh SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -source /opt/ic/bin/functions.sh - GENERATION= MINIMUM_CPU_SOCKETS=2 diff --git a/ic-os/components/setupos-scripts/install-guestos.sh b/ic-os/components/setupos-scripts/install-guestos.sh index 41ff1464eb8..3e24773234d 100755 --- a/ic-os/components/setupos-scripts/install-guestos.sh +++ b/ic-os/components/setupos-scripts/install-guestos.sh @@ -8,8 +8,6 @@ source /opt/ic/bin/functions.sh SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -source /opt/ic/bin/functions.sh - LV="/dev/mapper/hostlvm-guestos" function install_guestos() { diff --git a/ic-os/components/setupos-scripts/install-hostos.sh b/ic-os/components/setupos-scripts/install-hostos.sh index e45198be553..4557a02957c 100755 --- a/ic-os/components/setupos-scripts/install-hostos.sh +++ b/ic-os/components/setupos-scripts/install-hostos.sh @@ -8,8 +8,6 @@ source /opt/ic/bin/functions.sh SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -source /opt/ic/bin/functions.sh - function install_hostos() { echo "* Installing HostOS disk-image..." diff --git a/ic-os/components/setupos-scripts/setup-disk.sh b/ic-os/components/setupos-scripts/setup-disk.sh index 499ed52bba1..38437fcf20b 100755 --- a/ic-os/components/setupos-scripts/setup-disk.sh +++ b/ic-os/components/setupos-scripts/setup-disk.sh @@ -8,8 +8,6 @@ source /opt/ic/bin/functions.sh SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -source /opt/ic/bin/functions.sh - function purge_partitions() { echo "* Purging partitions..." diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index d3f1851ead3..43a80956335 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -10,8 +10,6 @@ SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" CONFIG_DIR="/config" -source /opt/ic/bin/functions.sh - function mount_config_partition() { echo "* Mounting hostOS config partition..." diff --git a/ic-os/components/setupos-scripts/setupos.sh b/ic-os/components/setupos-scripts/setupos.sh index 223b11cd4c2..67fcbeedc77 100755 --- a/ic-os/components/setupos-scripts/setupos.sh +++ b/ic-os/components/setupos-scripts/setupos.sh @@ -8,8 +8,6 @@ source /opt/ic/bin/functions.sh SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -source /opt/ic/bin/functions.sh - function start_setupos() { # Wait until login prompt appears sleep 5 From 0e1d9bd69e8366198f768712de6a95c253aa4133 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 4 Oct 2024 22:17:34 +0000 Subject: [PATCH 039/241] Reorganize sourcing in setupos scripts --- ic-os/components/setupos-scripts/check-config.sh | 6 +++--- ic-os/components/setupos-scripts/check-hardware.sh | 6 +++--- ic-os/components/setupos-scripts/check-network.sh | 6 +++--- ic-os/components/setupos-scripts/install-guestos.sh | 4 ++-- ic-os/components/setupos-scripts/install-hostos.sh | 4 ++-- ic-os/components/setupos-scripts/setup-disk.sh | 4 ++-- ic-os/components/setupos-scripts/setup-hostos-config.sh | 6 +++--- ic-os/components/setupos-scripts/setupos.sh | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ic-os/components/setupos-scripts/check-config.sh b/ic-os/components/setupos-scripts/check-config.sh index fa20310e9fc..cfeb10c2dcf 100644 --- a/ic-os/components/setupos-scripts/check-config.sh +++ b/ic-os/components/setupos-scripts/check-config.sh @@ -3,12 +3,12 @@ set -o nounset set -o pipefail -source /opt/ic/bin/config.sh -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + check_config_file() { echo "* Checking Config..." local CONFIG_FILE="/var/ic/config/config.json" diff --git a/ic-os/components/setupos-scripts/check-hardware.sh b/ic-os/components/setupos-scripts/check-hardware.sh index 3d77b7ac2cb..96e7eb2ba91 100644 --- a/ic-os/components/setupos-scripts/check-hardware.sh +++ b/ic-os/components/setupos-scripts/check-hardware.sh @@ -3,12 +3,12 @@ set -o nounset set -o pipefail -source /opt/ic/bin/config.sh -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + GENERATION= MINIMUM_CPU_SOCKETS=2 diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index 09049eaa2d8..d2056478d0f 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -3,12 +3,12 @@ set -o nounset set -o pipefail -source /opt/ic/bin/config.sh -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') diff --git a/ic-os/components/setupos-scripts/install-guestos.sh b/ic-os/components/setupos-scripts/install-guestos.sh index 3e24773234d..99fc322f119 100755 --- a/ic-os/components/setupos-scripts/install-guestos.sh +++ b/ic-os/components/setupos-scripts/install-guestos.sh @@ -3,11 +3,11 @@ set -o nounset set -o pipefail -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/functions.sh + LV="/dev/mapper/hostlvm-guestos" function install_guestos() { diff --git a/ic-os/components/setupos-scripts/install-hostos.sh b/ic-os/components/setupos-scripts/install-hostos.sh index 4557a02957c..942c3400387 100755 --- a/ic-os/components/setupos-scripts/install-hostos.sh +++ b/ic-os/components/setupos-scripts/install-hostos.sh @@ -3,11 +3,11 @@ set -o nounset set -o pipefail -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/functions.sh + function install_hostos() { echo "* Installing HostOS disk-image..." diff --git a/ic-os/components/setupos-scripts/setup-disk.sh b/ic-os/components/setupos-scripts/setup-disk.sh index 38437fcf20b..7f55b0523e2 100755 --- a/ic-os/components/setupos-scripts/setup-disk.sh +++ b/ic-os/components/setupos-scripts/setup-disk.sh @@ -3,11 +3,11 @@ set -o nounset set -o pipefail -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/functions.sh + function purge_partitions() { echo "* Purging partitions..." diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 43a80956335..0b5001c9454 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -3,13 +3,13 @@ set -o nounset set -o pipefail -source /opt/ic/bin/config.sh -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" CONFIG_DIR="/config" +source /opt/ic/bin/config.sh +source /opt/ic/bin/functions.sh + function mount_config_partition() { echo "* Mounting hostOS config partition..." diff --git a/ic-os/components/setupos-scripts/setupos.sh b/ic-os/components/setupos-scripts/setupos.sh index 67fcbeedc77..07af80ec6b7 100755 --- a/ic-os/components/setupos-scripts/setupos.sh +++ b/ic-os/components/setupos-scripts/setupos.sh @@ -3,11 +3,11 @@ set -o nounset set -o pipefail -source /opt/ic/bin/functions.sh - SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/functions.sh + function start_setupos() { # Wait until login prompt appears sleep 5 From cd15138942e2df0e88818b084ff24147cb39ae1d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 7 Oct 2024 18:10:00 +0000 Subject: [PATCH 040/241] Remove unnecessary setup-hostname logic --- .../setup-hostname/hostos/setup-hostname.sh | 54 ++++++------------- 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh index 5d5b7294663..ca5a41ca382 100755 --- a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh @@ -12,10 +12,6 @@ SCRIPT="$(basename $0)[$$]" # Get keyword arguments for argument in "${@}"; do case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; -f=* | --file=*) FILE="${argument#*=}" shift @@ -25,7 +21,6 @@ for argument in "${@}"; do Set Transient Or Persistent Hostname Arguments: - -c=, --config= optional: specify the config.ini configuration file (Default: /boot/config/config.ini) -f=, --file= optional: specify the file containing the node-id (Default: /boot/config/node-id) -h, --help show this help message and exit -t=, --type= mandatory: specify the node type (Examples: host, guest, boundary...) @@ -44,48 +39,31 @@ Arguments: done # Set arguments if undefined -CONFIG="${CONFIG:=/boot/config/config.ini}" FILE="${FILE:=/boot/config/node-id}" function validate_arguments() { - if [ "${CONFIG}" == "" -o "${FILE}" == "" -o "${TYPE}" == "" ]; then + if [ "${FILE}" == "" -o "${TYPE}" == "" ]; then $0 --help fi } -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; - "hostname") hostname="${value}" ;; - esac - done <"${CONFIG}" -} - function construct_hostname() { - if [ -z "${hostname}" ]; then - local mac=$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g') + local mac=$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g') - if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then - HOSTNAME=$(echo ${TYPE}-${mac}-$(cat ${FILE})) - write_log "Using hostname: ${HOSTNAME}" - write_metric "hostos_setup_hostname" \ - "1" \ - "HostOS setup hostname" \ - "gauge" - else - HOSTNAME=$(echo ${TYPE}-${mac}) - write_log "Using hostname: ${HOSTNAME}" - write_metric "hostos_setup_hostname" \ - "0" \ - "HostOS setup hostname" \ - "gauge" - fi + if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then + HOSTNAME=$(echo ${TYPE}-${mac}-$(cat ${FILE})) + write_log "Using hostname: ${HOSTNAME}" + write_metric "hostos_setup_hostname" \ + "1" \ + "HostOS setup hostname" \ + "gauge" else - HOSTNAME="${hostname}" + HOSTNAME=$(echo ${TYPE}-${mac}) + write_log "Using hostname: ${HOSTNAME}" + write_metric "hostos_setup_hostname" \ + "0" \ + "HostOS setup hostname" \ + "gauge" fi } @@ -103,9 +81,7 @@ function setup_hostname() { } function main() { - # Establish run order validate_arguments - read_variables construct_hostname setup_hostname } From 5385e7e87416d22485dc03252249d560e3725306 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 7 Oct 2024 18:13:08 +0000 Subject: [PATCH 041/241] Log config.json in log-config.sh --- ic-os/components/hostos-scripts/log-config/log-config.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/log-config/log-config.sh b/ic-os/components/hostos-scripts/log-config/log-config.sh index 5f8399c90b4..ad477f817d9 100644 --- a/ic-os/components/hostos-scripts/log-config/log-config.sh +++ b/ic-os/components/hostos-scripts/log-config/log-config.sh @@ -2,7 +2,6 @@ CONFIG_DIR="/boot/config" CONFIG="/boot/config/config.ini" -DEPLOYMENT="/boot/config/deployment.json" log_directory_structure() { local dir=$1 @@ -31,4 +30,3 @@ log_file_contents() { echo "Logging HostOS config partition" log_directory_structure "$CONFIG_DIR" log_file_contents "$CONFIG" -log_file_contents "$DEPLOYMENT" From e0af9fac0b4e50982039c816eb8a196bbf298f19 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 7 Oct 2024 18:17:01 +0000 Subject: [PATCH 042/241] Remove copying config.ini and deployment.json to HostOS --- .../setupos-scripts/setup-hostos-config.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 0b5001c9454..5ec99deb3a8 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -21,15 +21,6 @@ function mount_config_partition() { } function copy_config_files() { - # todo: remove copying of config.ini: - echo "* Copying 'config.ini' to hostOS config partition..." - if [ -f "${CONFIG_DIR}/config.ini" ]; then - cp ${CONFIG_DIR}/config.ini /media/ - log_and_halt_installation_on_error "${?}" "Unable to copy 'config.ini' to hostOS config partition." - else - log_and_halt_installation_on_error "1" "Configuration file 'config.ini' does not exist." - fi - echo "* Copying SSH authorized keys..." ssh_authorized_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') if [ -n "${ssh_authorized_keys}" ] && [ "${ssh_authorized_keys}" != "null" ]; then @@ -54,11 +45,6 @@ function copy_config_files() { echo >&2 "Warning: node_operator_private_key.pem does not exist, requiring HSM." fi - # todo: remove copying of config.ini: - echo "* Copying deployment.json to config partition..." - cp /data/deployment.json /media/ - log_and_halt_installation_on_error "${?}" "Unable to copy deployment.json to hostOS config partition." - echo "* Copying NNS public key to hostOS config partition..." nns_public_key_path=$(get_config_value '.icos_settings.nns_public_key_path') cp "${nns_public_key_path}" /media/ From 92c5213d08b1eccd488ca02dfe1ab55b51cb1d1d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 7 Oct 2024 18:17:33 +0000 Subject: [PATCH 043/241] Fix CONFIG name in log-config --- ic-os/components/hostos-scripts/log-config/log-config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/log-config/log-config.sh b/ic-os/components/hostos-scripts/log-config/log-config.sh index ad477f817d9..bf311cde304 100644 --- a/ic-os/components/hostos-scripts/log-config/log-config.sh +++ b/ic-os/components/hostos-scripts/log-config/log-config.sh @@ -1,7 +1,7 @@ #!/bin/bash CONFIG_DIR="/boot/config" -CONFIG="/boot/config/config.ini" +CONFIG="/boot/config/config.json" log_directory_structure() { local dir=$1 From 426c17653244bb22546a6fe1bec7a9cf5047440d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 7 Oct 2024 18:22:23 +0000 Subject: [PATCH 044/241] Remove unused HostOS config constants --- rs/ic_os/config/src/lib.rs | 2 -- rs/ic_os/os_tools/hostos_tool/src/main.rs | 12 +----------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 9a513e4161f..8df5ca564a8 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -19,8 +19,6 @@ pub static DEFAULT_SETUPOS_NODE_OPERATOR_PRIVATE_KEY_PATH: &str = pub static DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/config-hostos.json"; pub static DEFAULT_HOSTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config.json"; -pub static DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH: &str = "/boot/config/config.ini"; -pub static DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH: &str = "/boot/config/deployment.json"; pub fn serialize_and_write_config(path: &Path, config: &T) -> Result<()> { let serialized_config = diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 5f1f134d956..79327f5df6b 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -4,10 +4,7 @@ use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use config::types::{HostOSConfig, Ipv6Config}; -use config::{ - deserialize_config, DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH, DEFAULT_HOSTOS_CONFIG_OBJECT_PATH, - DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH, -}; +use config::{deserialize_config, DEFAULT_HOSTOS_CONFIG_OBJECT_PATH}; use network::generate_network_config; use network::ipv6::generate_ipv6_address; use network::mac_address::{generate_mac_address, get_ipmi_mac, FormattedMacAddress}; @@ -35,13 +32,6 @@ pub enum Commands { #[derive(Parser)] struct HostOSArgs { - #[arg(short, long, default_value_t = DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH.to_string(), value_name = "FILE")] - config: String, - - #[arg(short, long, default_value_t = DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH.to_string(), value_name = "FILE")] - /// deployment.json file path - deployment_file: String, - #[command(subcommand)] command: Option, } From e2d0d835eee20223371ab3a780875254d8fe7a46 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 7 Oct 2024 17:07:46 +0000 Subject: [PATCH 045/241] Replace fetch-mgmt-mac.sh with hostos_tool command --- .../setup-hostname/hostos/setup-hostname.sh | 2 +- .../dev-generate-guestos-config.sh | 3 +- .../generate-guestos-config.sh | 3 +- .../hostos-scripts/misc/fetch-mgmt-mac.sh | 69 ------------------- ic-os/components/hostos.bzl | 1 - rs/ic_os/os_tools/hostos_tool/src/main.rs | 27 +++++++- 6 files changed, 28 insertions(+), 77 deletions(-) delete mode 100755 ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh index ca5a41ca382..06029ba1fc3 100755 --- a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh @@ -48,7 +48,7 @@ function validate_arguments() { } function construct_hostname() { - local mac=$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g') + local mac=$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g') if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then HOSTNAME=$(echo ${TYPE}-${mac}-$(cat ${FILE})) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 998afe14bcf..0208f3ec2af 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -80,8 +80,7 @@ function assemble_config_media() { cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain}") fi - # todo: can I use the fetch-mgmt-mac in hostos tool? - cmd+=(--hostname "guest-$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g')") + cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_url "$nns_url") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index bc7f783c691..c973c912b06 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -79,8 +79,7 @@ function assemble_config_media() { cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain}") fi - # todo: can I use the fetch-mgmt-mac in hostos tool? - cmd+=(--hostname "guest-$(/opt/ic/bin/fetch-mgmt-mac.sh | sed 's/://g')") + cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_url "$nns_url") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") diff --git a/ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh b/ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh deleted file mode 100755 index 1dabbba53be..00000000000 --- a/ic-os/components/hostos-scripts/misc/fetch-mgmt-mac.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -set -e - -# Fetch the management MAC address of the physical machine. - -source /opt/ic/bin/logging.sh -source /opt/ic/bin/metrics.sh - -SCRIPT="$(basename $0)[$$]" -DEPLOYMENT="${DEPLOYMENT:=/boot/config/deployment.json}" - -# Get keyword arguments -for argument in "${@}"; do - case ${argument} in - -h | --help) - echo 'Usage: -Fetch Management MAC Address - -Arguments: - -h, --help show this help message and exit -' - exit 1 - ;; - *) - echo "Error: Argument is not supported." - exit 1 - ;; - esac -done - -# Fetch the management MAC address of the physical machine. -# The management MAC address will be used as unique key for: -# - Hostnames -# - IPv6 addresses -function fetch_mgmt_mac() { - MAC=$(ipmitool lan print | sed -e 's/^MAC Address.*\([0-9a-f:]\{17\}\)/\1/' -e t -e d) - - if [ "${MAC}" == "" ]; then - write_log "ERROR: Unable to determine MAC address." - write_metric "hostos_fetch_mgmt_mac" \ - "1" \ - "HostOS fetch management MAC address" \ - "gauge" - exit 1 - else - write_log "Unique management MAC address is: ${MAC}" - write_metric "hostos_fetch_mgmt_mac" \ - "0" \ - "HostOS fetch management MAC address" \ - "gauge" - fi - - echo "${MAC}" -} - -function main() { - # Establish run order - - MGMT_MAC=$(jq -r ".deployment.mgmt_mac" ${DEPLOYMENT}) - - if [ -z "${MGMT_MAC}" ] || [ "${MGMT_MAC}" = "null" ]; then - fetch_mgmt_mac - else - echo "${MGMT_MAC}" - fi -} - -main diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 424df253a0c..19ac88e2941 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -15,7 +15,6 @@ component_files = { Label("hostos-scripts/libvirt/setup-libvirt.sh"): "/opt/ic/bin/setup-libvirt.sh", Label("hostos-scripts/libvirt/setup-libvirt.service"): "/etc/systemd/system/setup-libvirt.service", Label("hostos-scripts/misc/setup-var.sh"): "/opt/ic/bin/setup-var.sh", - Label("hostos-scripts/misc/fetch-mgmt-mac.sh"): "/opt/ic/bin/fetch-mgmt-mac.sh", Label("hostos-scripts/misc/detect-first-boot.sh"): "/opt/ic/bin/detect-first-boot.sh", Label("hostos-scripts/monitoring/monitor-guestos.sh"): "/opt/ic/bin/monitor-guestos.sh", Label("hostos-scripts/monitoring/monitor-guestos.service"): "/etc/systemd/system/monitor-guestos.service", diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 79327f5df6b..c8c0ba045c9 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -20,14 +20,15 @@ pub enum Commands { /// systemd-networkd output directory output_directory: String, }, - GenerateMacAddress { + GenerateIpv6Address { #[arg(short, long, default_value = "HostOS")] node_type: String, }, - GenerateIpv6Address { + GenerateMacAddress { #[arg(short, long, default_value = "HostOS")] node_type: String, }, + FetchMacAddress {}, } #[derive(Parser)] @@ -167,6 +168,28 @@ pub fn main() -> Result<()> { println!("{}", generated_mac); Ok(()) } + Some(Commands::FetchMacAddress {}) => { + let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)) + .context(format!( + "Failed to get deployment settings for file: {}", + &opts.deployment_file + ))?; + eprintln!("Deployment config: {:?}", deployment_settings); + + let mgmt_mac = match deployment_settings.deployment.mgmt_mac { + Some(config_mac) => { + let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; + eprintln!( + "Using mgmt_mac address found in deployment.json: {}", + mgmt_mac + ); + mgmt_mac + } + None => get_ipmi_mac()?, + }; + println!("{}", mgmt_mac); + Ok(()) + } None => Err(anyhow!( "No subcommand specified. Run with '--help' for subcommands" )), From 0bfe363fad6d908058668952c326604c4f2adff2 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 7 Oct 2024 18:40:44 +0000 Subject: [PATCH 046/241] Fix hostos_tool after merge --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index c8c0ba045c9..10711d92a40 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -169,14 +169,15 @@ pub fn main() -> Result<()> { Ok(()) } Some(Commands::FetchMacAddress {}) => { - let deployment_settings = get_deployment_settings(Path::new(&opts.deployment_file)) - .context(format!( - "Failed to get deployment settings for file: {}", - &opts.deployment_file - ))?; - eprintln!("Deployment config: {:?}", deployment_settings); - - let mgmt_mac = match deployment_settings.deployment.mgmt_mac { + let hostos_config: HostOSConfig = + deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + + let mgmt_mac = match hostos_config + .icos_settings + .icos_dev_settings + .mgmt_mac + .as_ref() + { Some(config_mac) => { let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; eprintln!( From 7b0c77151641197cdf2a9fd3909ba5c098dd6cd7 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 8 Oct 2024 14:44:33 +0000 Subject: [PATCH 047/241] Revert nns_url renaming --- .../generate-guestos-config/dev-generate-guestos-config.sh | 4 ++-- .../generate-guestos-config/generate-guestos-config.sh | 4 ++-- ic-os/components/setupos-scripts/check-network.sh | 4 ++-- rs/ic_os/config/src/lib.rs | 2 +- rs/ic_os/config/src/main.rs | 2 +- rs/ic_os/config/src/types.rs | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 0208f3ec2af..f5385107ab0 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -62,7 +62,7 @@ function read_config_variables() { domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') - nns_url=$(get_config_value '.icos_settings.nns_url | join(",")') + nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') @@ -81,7 +81,7 @@ function assemble_config_media() { cmd+=(--domain "${domain}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") - cmd+=(--nns_url "$nns_url") + cmd+=(--nns_url "$nns_urls") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index c973c912b06..1f95d4e3e9f 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -62,7 +62,7 @@ function read_config_variables() { domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') - nns_url=$(get_config_value '.icos_settings.nns_url | join(",")') + nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') @@ -80,7 +80,7 @@ function assemble_config_media() { cmd+=(--domain "${domain}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") - cmd+=(--nns_url "$nns_url") + cmd+=(--nns_url "$nns_urls") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index d2056478d0f..d898f91dfdd 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -165,10 +165,10 @@ function ping_ipv6_gateway() { function query_nns_nodes() { echo "* Querying NNS nodes..." - local nns_url=($(get_config_value '.icos_settings.nns_url' | jq -r '.[]')) + local nns_urls=($(get_config_value '.icos_settings.nns_urls' | jq -r '.[]')) local success=false - for url in "${nns_url[@]}"; do + for url in "${nns_urls[@]}"; do # When running against testnets, we need to ignore self signed certs # with `--insecure`. This check is only meant to confirm from SetupOS # that NNS urls are reachable, so we do not mind that it is "weak". diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 8df5ca564a8..a024884056f 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -72,7 +72,7 @@ mod tests { let icos_settings = ICOSSettings { logging, nns_public_key_path: PathBuf::from("/path/to/key"), - nns_url: vec!["http://localhost".parse().unwrap()], + nns_urls: vec!["http://localhost".parse().unwrap()], hostname: "mainnet".to_string(), node_operator_private_key_path: None, ssh_authorized_keys_path: None, diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index c6be7a4fa8b..fa502743702 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -113,7 +113,7 @@ pub fn main() -> Result<()> { let icos_settings = ICOSSettings { logging, nns_public_key_path: nns_public_key_path.to_path_buf(), - nns_url: deployment_json_settings.nns.url.clone(), + nns_urls: deployment_json_settings.nns.url.clone(), hostname: deployment_json_settings.deployment.name.to_string(), node_operator_private_key_path: node_operator_private_key_path .exists() diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index a9d62c36908..1ca9963492a 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -87,7 +87,7 @@ pub struct ICOSSettings { /// This file must be a text file containing the public key of the NNS to be used. pub nns_public_key_path: PathBuf, /// The URL (HTTP) of the NNS node(s). - pub nns_url: Vec, + pub nns_urls: Vec, pub hostname: String, /// This file contains the Node Operator private key, /// which is registered with the NNS and used to sign the IC join request. From 63b0d12859dcb9172e4a0fac17d6250fc84fb4e1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 8 Oct 2024 14:45:53 +0000 Subject: [PATCH 048/241] Rename nns_url_list --- ic-os/components/setupos-scripts/check-network.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index d898f91dfdd..fec504ee315 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -165,10 +165,10 @@ function ping_ipv6_gateway() { function query_nns_nodes() { echo "* Querying NNS nodes..." - local nns_urls=($(get_config_value '.icos_settings.nns_urls' | jq -r '.[]')) + local nns_url_list=($(get_config_value '.icos_settings.nns_urls' | jq -r '.[]')) local success=false - for url in "${nns_urls[@]}"; do + for url in "${nns_url_list[@]}"; do # When running against testnets, we need to ignore self signed certs # with `--insecure`. This check is only meant to confirm from SetupOS # that NNS urls are reachable, so we do not mind that it is "weak". From a933b395bb2f169f61e44bcbacdc6d4c2c4990d8 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 9 Oct 2024 18:20:34 +0000 Subject: [PATCH 049/241] Add missing check-config.sh in setupos --- ic-os/components/setupos-scripts/setupos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ic-os/components/setupos-scripts/setupos.sh b/ic-os/components/setupos-scripts/setupos.sh index 07af80ec6b7..1c4d5645e58 100755 --- a/ic-os/components/setupos-scripts/setupos.sh +++ b/ic-os/components/setupos-scripts/setupos.sh @@ -39,6 +39,7 @@ main() { log_start "$(basename $0)" start_setupos /opt/ic/bin/check-setupos-age.sh + /opt/ic/bin/check-config.sh /opt/ic/bin/check-hardware.sh /opt/ic/bin/check-network.sh /opt/ic/bin/setup-disk.sh From 3f10503be07e28c050a033edac38c9cc41d31ed5 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 9 Oct 2024 18:24:03 +0000 Subject: [PATCH 050/241] Add check-config file description --- ic-os/components/setupos-scripts/check-config.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ic-os/components/setupos-scripts/check-config.sh b/ic-os/components/setupos-scripts/check-config.sh index cfeb10c2dcf..273b2938319 100644 --- a/ic-os/components/setupos-scripts/check-config.sh +++ b/ic-os/components/setupos-scripts/check-config.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# check-config.sh verifies the existence of the configuration JSON file created by config.service, halting the installation if not found. + set -o nounset set -o pipefail From b8317dbe234bf6198dbd800290eb8e39fb9cf20b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 9 Oct 2024 18:25:41 +0000 Subject: [PATCH 051/241] Update check-config.sh comment --- ic-os/components/setupos-scripts/check-config.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ic-os/components/setupos-scripts/check-config.sh b/ic-os/components/setupos-scripts/check-config.sh index 273b2938319..dcc08525a09 100644 --- a/ic-os/components/setupos-scripts/check-config.sh +++ b/ic-os/components/setupos-scripts/check-config.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# check-config.sh verifies the existence of the configuration JSON file created by config.service, halting the installation if not found. +# check-config.sh verifies the existence of the configuration JSON file created by config.service, +# halting the installation if not found. set -o nounset set -o pipefail From 615d3d66c0f24f932146a7be18b41c8c806fb875 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 9 Oct 2024 20:20:58 +0000 Subject: [PATCH 052/241] Add guestos config object to guestos config --- .../build-bootstrap-config-image.sh | 11 ++++++ .../guestos/bootstrap-ic-node.sh | 2 +- ic-os/hostos/defs.bzl | 1 + rs/ic_os/config/src/lib.rs | 1 + rs/ic_os/config/src/main.rs | 39 +++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index ec1f21cfd1e..4ac13f943ed 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -275,6 +275,17 @@ EOF cp "${NODE_OPERATOR_PRIVATE_KEY}" "${BOOTSTRAP_TMPDIR}/node_operator_private_key.pem" fi + # Create guestos config.json (but not break testing) + if [ -f "/boot/config/config.json" ]; then + /opt/ic/bin/config generate-guestos-config + echo "* Copying 'config-guestos.json' to GuestOS config partition..." + if [ -f "/boot/config/config-guestos.json" ]; then + cp /boot/config/config-guestos.json "${BOOTSTRAP_TMPDIR}/config.json" + # else + # todo: fix ERROR + fi + fi + tar cf "${OUT_FILE}" \ --sort=name \ --owner=root:0 \ diff --git a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh index b2b928af869..7bc91d28021 100755 --- a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh @@ -102,7 +102,7 @@ function process_bootstrap() { # stash the following configuration files to config store # note: keep this list in sync with configurations supported in build-bootstrap-config-image.sh - for FILE in filebeat.conf network.conf nns.conf backup.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf jaeger_addr.conf socks_proxy.conf; do + for FILE in filebeat.conf network.conf nns.conf backup.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf jaeger_addr.conf socks_proxy.conf config.json; do if [ -e "${TMPDIR}/${FILE}" ]; then echo "Setting up ${FILE}" cp "${TMPDIR}/${FILE}" "${CONFIG_ROOT}/${FILE}" diff --git a/ic-os/hostos/defs.bzl b/ic-os/hostos/defs.bzl index e6c1af9dcc3..a55f747f0b1 100644 --- a/ic-os/hostos/defs.bzl +++ b/ic-os/hostos/defs.bzl @@ -31,6 +31,7 @@ def image_deps(mode, _malicious = False): # additional files to install "//rs/ic_os/release:vsock_host": "/opt/ic/bin/vsock_host:0755", "//rs/ic_os/release:hostos_tool": "/opt/ic/bin/hostos_tool:0755", + "//rs/ic_os/release:config": "/opt/ic/bin/config:0755", "//rs/ic_os/release:metrics-proxy": "/opt/ic/bin/metrics-proxy:0755", # additional libraries to install diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index a024884056f..eebbed0263a 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -19,6 +19,7 @@ pub static DEFAULT_SETUPOS_NODE_OPERATOR_PRIVATE_KEY_PATH: &str = pub static DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/config-hostos.json"; pub static DEFAULT_HOSTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config.json"; +pub static DEFAULT_HOSTOS_GUESTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config-guestos.json"; pub fn serialize_and_write_config(path: &Path, config: &T) -> Result<()> { let serialized_config = diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index fa502743702..0b48068e308 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -37,6 +37,13 @@ pub enum Commands { #[arg(long, default_value = config::DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH, value_name = "config-hostos.json")] hostos_config_json_path: PathBuf, }, + /// Creates GuestOSConfig object from existing HostOS config.json file + GenerateGuestosConfig { + #[arg(long, default_value = config::DEFAULT_HOSTOS_CONFIG_OBJECT_PATH, value_name = "config.json")] + hostos_config_json_path: PathBuf, + #[arg(long, default_value = config::DEFAULT_HOSTOS_GUESTOS_CONFIG_OBJECT_PATH, value_name = "config-guestos.json")] + guestos_config_json_path: PathBuf, + }, } #[derive(Parser)] @@ -195,6 +202,38 @@ pub fn main() -> Result<()> { Ok(()) } + Some(Commands::GenerateGuestosConfig { + hostos_config_json_path, + guestos_config_json_path, + }) => { + let hostos_config_json_path = Path::new(&hostos_config_json_path); + + let hostos_config: HostOSConfig = + serde_json::from_reader(File::open(hostos_config_json_path)?)?; + + // update select file paths for GuestOS + let mut guestos_icos_settings = hostos_config.icos_settings; + let guestos_config_path = Path::new("/boot/config"); + if let Some(ref mut path) = guestos_icos_settings.ssh_authorized_keys_path { + *path = guestos_config_path.join("accounts_ssh_authorized_keys"); + } + + let guestos_config = GuestOSConfig { + network_settings: hostos_config.network_settings, + icos_settings: guestos_icos_settings, + guestos_settings: hostos_config.guestos_settings, + }; + + let guestos_config_json_path = Path::new(&guestos_config_json_path); + serialize_and_write_config(guestos_config_json_path, &guestos_config)?; + + println!( + "GuestOSConfig has been written to {}", + guestos_config_json_path.display() + ); + + Ok(()) + } None => Ok(()), } } From 71079e27cccb21034d4677bb9f11ae0cf17d733c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 9 Oct 2024 21:00:21 +0000 Subject: [PATCH 053/241] Add guestos config.sh and reorganize and rename config scripts --- ic-os/components/guestos.bzl | 1 + ic-os/components/hostos.bzl | 2 +- ic-os/components/misc/config/{hostos-config.sh => config.sh} | 0 .../misc/config/{setupos-config.sh => setupos/config.sh} | 0 ic-os/components/setupos.bzl | 2 +- 5 files changed, 3 insertions(+), 2 deletions(-) rename ic-os/components/misc/config/{hostos-config.sh => config.sh} (100%) rename ic-os/components/misc/config/{setupos-config.sh => setupos/config.sh} (100%) diff --git a/ic-os/components/guestos.bzl b/ic-os/components/guestos.bzl index d1e40f2bc88..b8110b6d39d 100644 --- a/ic-os/components/guestos.bzl +++ b/ic-os/components/guestos.bzl @@ -47,6 +47,7 @@ component_files = { Label("init/setup-lvs/guestos/setup-lvs.sh"): "/opt/ic/bin/setup-lvs.sh", # misc + Label("misc/config/config.sh"): "/opt/ic/bin/config.sh", Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", Label("misc/metrics.sh"): "/opt/ic/bin/metrics.sh", Label("misc/serial-getty@/guestos/serial-getty@.service"): "/etc/systemd/system/serial-getty@.service", diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 6a0243f8a55..339323557e9 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -49,7 +49,7 @@ component_files = { Label("early-boot/initramfs-tools/hostos/set-machine-id/set-machine-id"): "/etc/initramfs-tools/scripts/init-bottom/set-machine-id/set-machine-id", # misc - Label("misc/config/hostos-config.sh"): "/opt/ic/bin/config.sh", + Label("misc/config/config.sh"): "/opt/ic/bin/config.sh", Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", Label("misc/metrics.sh"): "/opt/ic/bin/metrics.sh", Label("misc/vsock/vsock-agent.service"): "/etc/systemd/system/vsock-agent.service", diff --git a/ic-os/components/misc/config/hostos-config.sh b/ic-os/components/misc/config/config.sh similarity index 100% rename from ic-os/components/misc/config/hostos-config.sh rename to ic-os/components/misc/config/config.sh diff --git a/ic-os/components/misc/config/setupos-config.sh b/ic-os/components/misc/config/setupos/config.sh similarity index 100% rename from ic-os/components/misc/config/setupos-config.sh rename to ic-os/components/misc/config/setupos/config.sh diff --git a/ic-os/components/setupos.bzl b/ic-os/components/setupos.bzl index eb956d26f1a..a0c73cd053d 100644 --- a/ic-os/components/setupos.bzl +++ b/ic-os/components/setupos.bzl @@ -28,7 +28,7 @@ component_files = { # misc Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", - Label("misc/config/setupos-config.sh"): "/opt/ic/bin/config.sh", + Label("misc/config/setupos/config.sh"): "/opt/ic/bin/config.sh", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", Label("misc/chrony/chrony-var.service"): "/etc/systemd/system/chrony-var.service", Label("misc/serial-getty@/setupos/serial-getty@.service"): "/etc/systemd/system/serial-getty@.service", From 455ad95e0031b327fd331cf7d0954e5d5748855b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 9 Oct 2024 22:56:40 +0000 Subject: [PATCH 054/241] Remove conf scripts from generate-replica-config inputs except network.conf and malicious.conf --- .../components/ic/generate-replica-config.sh | 129 +++++------------- ic-os/components/ic/ic-replica.service | 2 +- ic-os/components/ic/ic.json5.template | 2 +- 3 files changed, 36 insertions(+), 97 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 22bcc739986..eb60effd743 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -3,19 +3,19 @@ # Substitute correct configuration parameters into ic.json5. Will take IP addresses # from configuration file or from network interfaces. +source /opt/ic/bin/config.sh + function usage() { cat <"${OUT_FILE}" diff --git a/ic-os/components/ic/ic-replica.service b/ic-os/components/ic/ic-replica.service index 234214b911b..30336ce9a3d 100644 --- a/ic-os/components/ic/ic-replica.service +++ b/ic-os/components/ic/ic-replica.service @@ -28,7 +28,7 @@ Environment=RUST_BACKTRACE=1 # Remember to update 'rs/default.nix' for nix-shell users # Remember to update 'src/dfx/src/actors/replica.rs' in the sdk repo for dfx users Environment=RUST_MIN_STACK=8192000 -ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -t /boot/config/jaeger_addr.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 +ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -m /boot/config/malicious_behavior.conf -c /boot/config/config.json -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 ExecStart=/opt/ic/bin/orchestrator --replica-binary-dir /var/lib/ic/data/images --cup-dir /var/lib/ic/data/cups --replica-config-file /run/ic-node/config/ic.json5 --enable-provisional-registration --ic-binary-directory /opt/ic/bin --orchestrator-data-directory /var/lib/ic/data/orchestrator --version-file /opt/ic/share/version.txt LimitNOFILE=16777216 Restart=always diff --git a/ic-os/components/ic/ic.json5.template b/ic-os/components/ic/ic.json5.template index 1e4336bd326..d47b58fc2ca 100644 --- a/ic-os/components/ic/ic.json5.template +++ b/ic-os/components/ic/ic.json5.template @@ -504,7 +504,7 @@ table ip6 filter {\n\ }, registration: { - nns_url: "{{ nns_url }}", + nns_url: "{{ nns_urls }}", nns_pub_key_pem: "/var/lib/ic/data/nns_public_key.pem", node_operator_pem: "/var/lib/ic/data/node_operator_private_key.pem" }, From f3a8adb0eeac8be37231d101d699fb6ffa1ed045 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 10 Oct 2024 13:51:55 +0000 Subject: [PATCH 055/241] Fix pre-commit --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 4ac13f943ed..25baf577969 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -281,7 +281,7 @@ EOF echo "* Copying 'config-guestos.json' to GuestOS config partition..." if [ -f "/boot/config/config-guestos.json" ]; then cp /boot/config/config-guestos.json "${BOOTSTRAP_TMPDIR}/config.json" - # else + # else # todo: fix ERROR fi fi From 29721d1c07c857916224ee09e10a4d98d115b055 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 10 Oct 2024 16:13:51 +0000 Subject: [PATCH 056/241] Make config_file optional for ic-crypto-csp service to run successfully --- ic-os/components/ic/generate-replica-config.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index f269bea8817..c3b21c4bd8a 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -145,8 +145,11 @@ fi if [ "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" != "" -a -e "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" ]; then read_malicious_behavior_variables "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" fi +if [ "${CONFIG_FILE}" != "" -a -e "${CONFIG_FILE}" ]; then + read_config_variables "${CONFIG_FILE}" +fi + -read_config_variables "${CONFIG_FILE}" INTERFACE=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';')) IPV6_ADDRESS="${ipv6_address%/*}" From c0c1f7dd2c37daa4b04ddd23dbc722c9fee2e8c1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 10 Oct 2024 16:24:59 +0000 Subject: [PATCH 057/241] Fix ipv4 address setting --- ic-os/components/ic/generate-replica-config.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index c3b21c4bd8a..948ce445d8d 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -61,6 +61,7 @@ function get_if_address_retries() { function read_config_variables() { ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') domain=$(get_config_value '.network_settings.ipv4_config.domain') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') @@ -154,7 +155,11 @@ fi INTERFACE=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';')) IPV6_ADDRESS="${ipv6_address%/*}" IPV6_ADDRESS="${IPV6_ADDRESS:-$(get_if_address_retries 6 ${INTERFACE} 12)}" -IPV4_ADDRESS="${ipv4_address:-}" +if [[ -n "$ipv4_address" && "$ipv4_address" != "null" && -n "$ipv4_prefix_length" && "$ipv4_prefix_length" != "null" ]]; then + IPV4_ADDRESS="${ipv4_address}/${ipv4_prefix_length}" +else + IPV4_ADDRESS="" +fi IPV4_GATEWAY="${ipv4_gateway:-}" DOMAIN="${domain:-}" NNS_URLS="${nns_urls:-http://[::1]:8080}" From da38336455758d1e585b7c8fa04406cc7e89856d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 10 Oct 2024 16:54:25 +0000 Subject: [PATCH 058/241] Fix pre-commit --- ic-os/components/ic/generate-replica-config.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 948ce445d8d..1524666ed30 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -150,15 +150,13 @@ if [ "${CONFIG_FILE}" != "" -a -e "${CONFIG_FILE}" ]; then read_config_variables "${CONFIG_FILE}" fi - - INTERFACE=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';')) IPV6_ADDRESS="${ipv6_address%/*}" IPV6_ADDRESS="${IPV6_ADDRESS:-$(get_if_address_retries 6 ${INTERFACE} 12)}" if [[ -n "$ipv4_address" && "$ipv4_address" != "null" && -n "$ipv4_prefix_length" && "$ipv4_prefix_length" != "null" ]]; then - IPV4_ADDRESS="${ipv4_address}/${ipv4_prefix_length}" + IPV4_ADDRESS="${ipv4_address}/${ipv4_prefix_length}" else - IPV4_ADDRESS="" + IPV4_ADDRESS="" fi IPV4_GATEWAY="${ipv4_gateway:-}" DOMAIN="${domain:-}" From 3b6ace5a11b6802eb00b7acb1846fcd130e15a04 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 10 Oct 2024 18:12:03 +0000 Subject: [PATCH 059/241] Make config non-optional to fix ic-crypto-csp failure --- ic-os/components/ic/generate-replica-config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 1524666ed30..a9f2af406fa 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -135,7 +135,7 @@ while getopts "m:n:c:i:o:" OPT; do esac done -if [ "${CONFIG_FILE}" == "" -o "${IN_FILE}" == "" -o "${OUT_FILE}" == "" ]; then +if [ "${IN_FILE}" == "" -o "${OUT_FILE}" == "" ]; then usage exit 1 fi From 1325e007d1e3efb0303a68bfc43b20b12c1523d1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 10 Oct 2024 21:49:11 +0000 Subject: [PATCH 060/241] Remove config input to generate-replica-config --- ic-os/components/ic/generate-replica-config.sh | 11 +++-------- ic-os/components/ic/ic-replica.service | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index a9f2af406fa..4f67fe69e68 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -8,11 +8,10 @@ source /opt/ic/bin/config.sh function usage() { cat < Date: Thu, 10 Oct 2024 22:13:57 +0000 Subject: [PATCH 061/241] Use config object for generate-filebeat-config.sh --- .../monitoring/filebeat/README.adoc | 8 ----- .../monitoring/filebeat/filebeat.service | 4 +-- .../filebeat/generate-filebeat-config.sh | 35 +++++-------------- 3 files changed, 10 insertions(+), 37 deletions(-) delete mode 100644 ic-os/components/monitoring/filebeat/README.adoc diff --git a/ic-os/components/monitoring/filebeat/README.adoc b/ic-os/components/monitoring/filebeat/README.adoc deleted file mode 100644 index 47a67650ec9..00000000000 --- a/ic-os/components/monitoring/filebeat/README.adoc +++ /dev/null @@ -1,8 +0,0 @@ -= Filebeat configuration - -The Filebeat configuration is performed using a file +filebeat.conf+ in -the bootstrap tarball. It must contain lines of "key=value= statements, -with the following keys supported: - -* elasticsearch_hosts: space-separated list of logging hosts -* elasticsearch_tags: space-separated list of tags diff --git a/ic-os/components/monitoring/filebeat/filebeat.service b/ic-os/components/monitoring/filebeat/filebeat.service index 8521cb392de..709b3f64864 100644 --- a/ic-os/components/monitoring/filebeat/filebeat.service +++ b/ic-os/components/monitoring/filebeat/filebeat.service @@ -10,15 +10,13 @@ Wants=bootstrap-ic-node.service # We must wait for var to be mounted over before interacting with it After=var.mount Wants=var.mount -# Only start Filebeat if configuration file exists -ConditionPathExists=/boot/config/filebeat.conf [Service] User=filebeat Group=filebeat Environment="GODEBUG='madvdontneed=1'" ExecStartPre=+/opt/ic/bin/setup-filebeat-permissions.sh -ExecStartPre=+/opt/ic/bin/generate-filebeat-config.sh -j /boot/config/filebeat.conf -i /etc/filebeat/filebeat.yml.template -o /run/ic-node/etc/filebeat/filebeat.yml +ExecStartPre=+/opt/ic/bin/generate-filebeat-config.sh -i /etc/filebeat/filebeat.yml.template -o /run/ic-node/etc/filebeat/filebeat.yml ExecStart=/usr/local/bin/filebeat --environment systemd -e --path.home /var/lib/filebeat --path.config /run/ic-node/etc/filebeat --path.data /var/lib/filebeat --path.logs /var/log/filebeat Restart=always diff --git a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh index 8eb5841c8ff..99df7491fea 100755 --- a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh +++ b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh @@ -2,36 +2,24 @@ # Substitute correct configuration parameters into filebeat.yml. +source /opt/ic/bin/config.sh + function usage() { cat <"${OUT_FILE}" fi -if [ "${ELASTICSEARCH_TAGS}" != "" ]; then +if [ "${ELASTICSEARCH_TAGS}" != "" ] && ["${ELASTICSEARCH_TAGS}" != "null"]; then # Covert string into comma separated array elasticsearch_tags_array=$(for tag in ${ELASTICSEARCH_TAGS}; do echo -n "\"${tag}\", "; done | sed -E "s@, \$@@g") sed -e "s@#{{ elasticsearch_tags }}@tags: [${elasticsearch_tags_array}]@" -i "${OUT_FILE}" From 1cc5d808e38fd98d577be29cd82f6f0136fab9a4 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 10 Oct 2024 22:33:29 +0000 Subject: [PATCH 062/241] Move generate-btc-adapter-config and generate-https-outcalls-adapter-config to use config object --- .../generate-btc-adapter-config.sh | 48 ++++++------------- .../ic-btc-mainnet-adapter.service | 5 +- .../ic-btc-testnet-adapter.service | 3 +- .../generate-https-outcalls-adapter-config.sh | 26 ++++------ .../ic-https-outcalls-adapter.service | 2 +- 5 files changed, 26 insertions(+), 58 deletions(-) diff --git a/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh b/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh index 888415ff806..ebafed8059a 100755 --- a/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh +++ b/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh @@ -5,37 +5,22 @@ # # Arguments: # - $1: Name of the file to be read. -function read_bitcoind_addr_variables() { - while IFS="=" read -r key value; do - case "$key" in - "bitcoind_addr") bitcoind_addr="${value}" ;; - esac - done <"$1" -} -# Reads the socks proxy config file. The file must be of the form "key=value". -# The file should only contain the key `socks_proxy`. All other keys are ignored. -# -# Arguments: -# - $1: Name of the file to be read. -function read_socks_proxy() { - while IFS="=" read -r key value; do - case "$key" in - "socks_proxy") SOCKS_PROXY="${value}" ;; - esac - done <"$1" +source /opt/ic/bin/config.sh + +function read_config_variables() { + config_bitcoind_addr=$(get_config_value '.guestos_settings.guestos_dev_settings.bitcoind_addr') + config_socks_proxy=$(get_config_value '.guestos_settings.guestos_dev_settings.socks_proxy') } function usage() { cat < Date: Thu, 10 Oct 2024 22:33:49 +0000 Subject: [PATCH 063/241] Remove outstanding guestos conf files --- .../build-bootstrap-config-image.sh | 25 ------------------- .../guestos/bootstrap-ic-node.sh | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 25baf577969..ad1f2fd18c4 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -228,37 +228,12 @@ ${IPV4_ADDRESS:+ipv4_address=$IPV4_ADDRESS} ${IPV4_GATEWAY:+ipv4_gateway=$IPV4_GATEWAY} ${DOMAIN:+domain=$DOMAIN} EOF - if [ "${ELASTICSEARCH_HOSTS}" != "" ]; then - echo "elasticsearch_hosts=$ELASTICSEARCH_HOSTS" >"${BOOTSTRAP_TMPDIR}/filebeat.conf" - fi - if [ "${ELASTICSEARCH_TAGS}" != "" ]; then - echo "elasticsearch_tags=$ELASTICSEARCH_TAGS" >>"${BOOTSTRAP_TMPDIR}/filebeat.conf" - fi if [ "${NNS_PUBLIC_KEY}" != "" ]; then cp "${NNS_PUBLIC_KEY}" "${BOOTSTRAP_TMPDIR}/nns_public_key.pem" fi - if [ "${NNS_URL}" != "" ]; then - echo "nns_url=${NNS_URL}" >"${BOOTSTRAP_TMPDIR}/nns.conf" - fi - if [ "${BACKUP_RETENTION_TIME_SECS}" != "" ] || [ "${BACKUP_PURGING_INTERVAL_SECS}" != "" ]; then - echo "backup_retention_time_secs=${BACKUP_RETENTION_TIME_SECS}" >"${BOOTSTRAP_TMPDIR}/backup.conf" - echo "backup_puging_interval_secs=${BACKUP_PURGING_INTERVAL_SECS}" >>"${BOOTSTRAP_TMPDIR}/backup.conf" - fi if [ "${MALICIOUS_BEHAVIOR}" != "" ]; then echo "malicious_behavior=${MALICIOUS_BEHAVIOR}" >"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" fi - if [ "${QUERY_STATS_EPOCH_LENGTH}" != "" ]; then - echo "query_stats_epoch_length=${QUERY_STATS_EPOCH_LENGTH}" >"${BOOTSTRAP_TMPDIR}/query_stats.conf" - fi - if [ "${BITCOIND_ADDR}" != "" ]; then - echo "bitcoind_addr=${BITCOIND_ADDR}" >"${BOOTSTRAP_TMPDIR}/bitcoind_addr.conf" - fi - if [ "${JAEGER_ADDR}" != "" ]; then - echo "jaeger_addr=http://${JAEGER_ADDR}" >"${BOOTSTRAP_TMPDIR}/jaeger_addr.conf" - fi - if [ "${SOCKS_PROXY}" != "" ]; then - echo "socks_proxy=${SOCKS_PROXY}" >"${BOOTSTRAP_TMPDIR}/socks_proxy.conf" - fi if [ "${IC_CRYPTO}" != "" ]; then cp -r "${IC_CRYPTO}" "${BOOTSTRAP_TMPDIR}/ic_crypto" fi diff --git a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh index 7bc91d28021..0ca1f22480a 100755 --- a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh @@ -102,7 +102,7 @@ function process_bootstrap() { # stash the following configuration files to config store # note: keep this list in sync with configurations supported in build-bootstrap-config-image.sh - for FILE in filebeat.conf network.conf nns.conf backup.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf jaeger_addr.conf socks_proxy.conf config.json; do + for FILE in network.conf malicious_behavior.conf config.json; do if [ -e "${TMPDIR}/${FILE}" ]; then echo "Setting up ${FILE}" cp "${TMPDIR}/${FILE}" "${CONFIG_ROOT}/${FILE}" From 7204e1889a3225c3cbc61c379caa1af6eb463d05 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 18:21:03 +0000 Subject: [PATCH 064/241] Remove outdated comment --- .../ic/ic-btc-adapter/generate-btc-adapter-config.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh b/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh index ebafed8059a..c81fa76e40e 100755 --- a/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh +++ b/ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh @@ -74,14 +74,7 @@ if [ "${OUT_FILE}" == "" ]; then exit 1 fi -# BITCOIND_ADDR indicates that we are in system test environment. No socks proxy needed. -# bitcoin_addr.conf should be formatted like this: key 'bitcoind_addr', comma separated values, NO "" around addresses, NO trailing ',' AND spaces -# Example: bitcoind_addr=seed.bitcoin.sipa.be,regtest.random.me,regtest.random.org -# -# Bash explanation: -# ${bitcoind_addr:+\"${bitcoind_addr//,/\",\"}\"} -# ${parameter:+word}: If parameter is null or unset, nothing is substituted, otherwise the expansion of word is substituted. -# word: \"${bitcoind_addr//,/\",\"}\" Adds surrounding "" and matches and replaces all ',' with '","' +# config_bitcoind_addr indicates that we are in system test environment. No socks proxy needed. if [ "${config_bitcoind_addr}" != "" ] && [ "${config_bitcoind_addr}" != "null" ]; then echo '{ "network": "regtest", From c379217e683332e6950a1e053820b76d4d652f10 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 18:34:10 +0000 Subject: [PATCH 065/241] Update setup-ssh-account-keys to use the config object --- .../setup-ssh-account-keys.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh b/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh index d1106e2373d..18731902bfa 100755 --- a/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh +++ b/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh @@ -2,6 +2,12 @@ set -e +source /opt/ic/bin/config.sh + +read_config_variables() { + authorized_ssh_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') +} + copy_ssh_keys() { local SOURCE_FILE="$1" local DEST_FILE="$2" @@ -11,6 +17,8 @@ copy_ssh_keys() { fi } +read_config_variables + for ACCOUNT in backup readonly admin; do HOMEDIR=$(getent passwd "${ACCOUNT}" | cut -d: -f6) GROUP=$(id -ng "${ACCOUNT}") @@ -18,12 +26,10 @@ for ACCOUNT in backup readonly admin; do mkdir -p "${HOMEDIR}/.ssh" chmod 700 "${HOMEDIR}" "${HOMEDIR}/.ssh" - GUESTOS_AUTHORIZED_SSH_KEYS="/boot/config/accounts_ssh_authorized_keys/${ACCOUNT}" - HOSTOS_AUTHORIZED_SSH_KEYS="/boot/config/ssh_authorized_keys/${ACCOUNT}" + AUTHORIZED_SSH_KEYS="${authorized_ssh_keys}/${ACCOUNT}" AUTHORIZED_KEYS_FILE="${HOMEDIR}/.ssh/authorized_keys" - copy_ssh_keys "${GUESTOS_AUTHORIZED_SSH_KEYS}" "${AUTHORIZED_KEYS_FILE}" - copy_ssh_keys "${HOSTOS_AUTHORIZED_SSH_KEYS}" "${AUTHORIZED_KEYS_FILE}" + copy_ssh_keys "${AUTHORIZED_SSH_KEYS}" "${AUTHORIZED_KEYS_FILE}" chown -R "${ACCOUNT}:${GROUP}" "${HOMEDIR}" restorecon -r "${HOMEDIR}" From 4b0c75c7d47fb96055b9e3217067d2a73fd1a683 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 18:49:52 +0000 Subject: [PATCH 066/241] Remove old config from build-bootstrap --- .../build-bootstrap-config-image.sh | 71 +------------------ 1 file changed, 1 insertion(+), 70 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index ad1f2fd18c4..c396cac5f2f 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -53,20 +53,6 @@ options may be specified: material generated by ic-prep. Typically, this is IC_PREP_OUT_PATH/ic_registry_local_store - --elasticsearch_hosts hosts - Logging hosts to use. Can be multiple hosts separated by space (make sure - to quote the argument string so it appears as a single argument to the - script, e.g. --elasticsearch_hosts "h1.domain.tld h2.domain.tld"). - - --elasticsearch_tags tags - Tags to be used by Filebeat. Can be multiple tags separated by space - (make sure to quote the argument string so it appears as a single argument - to the script, e.g. --elasticsearch_tags "testnet1 slo") - - --nns_url url - URL of NNS nodes for sign up or registry access. Can be multiple nodes - separated by commas. - --nns_public_key path NNS public key file. @@ -81,35 +67,12 @@ options may be specified: --node_operator_private_key path Should point to a file containing a Node Provider private key PEM. - --backup_retention_time seconds - How long the backed up consensus artifacts should stay on the spool - before they get purged. - - --backup_puging_interval seconds - How often the backup purging should be executed. - --malicious_behavior malicious_behavior A JSON-object that describes the malicious behavior activated on the node. This is only used for testing. The Json-object corresponds to this Rust-structure: ic_types::malicious_behaviour::MaliciousBehaviour - - --query_stats_epoch_length length - The length of the epoch in seconds. To be used in - systems tests only. - - --bitcoind_addr address - The IP address of a running bitcoind instance. To be used in - systems tests only. - - --jaeger_addr address - The IP address of a running Jaeger Collector instance. To be used in - systems tests only. - - --socks_proxy url - The URL of the socks proxy to use. To be used in - systems tests only. EOF } @@ -122,14 +85,9 @@ function build_ic_bootstrap_tar() { local IPV6_ADDRESS IPV6_GATEWAY DOMAIN HOSTNAME local IC_CRYPTO IC_STATE IC_REGISTRY_LOCAL_STORE - local NNS_URL NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY - local BACKUP_RETENTION_TIME_SECS BACKUP_PURGING_INTERVAL_SECS - local ELASTICSEARCH_HOSTS ELASTICSEARCH_TAGS + local NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY local ACCOUNTS_SSH_AUTHORIZED_KEYS local MALICIOUS_BEHAVIOR - local QUERY_STATS_EPOCH_LENGTH - local BITCOIND_ADDR - local JAEGER_ADDR while true; do if [ $# == 0 ]; then @@ -164,15 +122,6 @@ function build_ic_bootstrap_tar() { --ic_registry_local_store) IC_REGISTRY_LOCAL_STORE="$2" ;; - --elasticsearch_hosts) - ELASTICSEARCH_HOSTS="$2" - ;; - --elasticsearch_tags) - ELASTICSEARCH_TAGS="$2" - ;; - --nns_url) - NNS_URL="$2" - ;; --nns_public_key) NNS_PUBLIC_KEY="$2" ;; @@ -182,27 +131,9 @@ function build_ic_bootstrap_tar() { --node_operator_private_key) NODE_OPERATOR_PRIVATE_KEY="$2" ;; - --backup_retention_time) - BACKUP_RETENTION_TIME_SECS="$2" - ;; - --backup_puging_interval) - BACKUP_PURGING_INTERVAL_SECS="$2" - ;; --malicious_behavior) MALICIOUS_BEHAVIOR="$2" ;; - --query_stats_epoch_length) - QUERY_STATS_EPOCH_LENGTH="$2" - ;; - --bitcoind_addr) - BITCOIND_ADDR="$2" - ;; - --jaeger_addr) - JAEGER_ADDR="$2" - ;; - --socks_proxy) - SOCKS_PROXY="$2" - ;; *) echo "Unrecognized option: $1" usage From f7c6379a342d66a58d23d0dfdf3210ce48132c6b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 18:56:28 +0000 Subject: [PATCH 067/241] Clean up build-bootstrap-config-image.sh --- .../build-bootstrap-config-image.sh | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index c396cac5f2f..b287612537e 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -162,6 +162,9 @@ EOF if [ "${NNS_PUBLIC_KEY}" != "" ]; then cp "${NNS_PUBLIC_KEY}" "${BOOTSTRAP_TMPDIR}/nns_public_key.pem" fi + if [ "${NODE_OPERATOR_PRIVATE_KEY}" != "" ]; then + cp "${NODE_OPERATOR_PRIVATE_KEY}" "${BOOTSTRAP_TMPDIR}/node_operator_private_key.pem" + fi if [ "${MALICIOUS_BEHAVIOR}" != "" ]; then echo "malicious_behavior=${MALICIOUS_BEHAVIOR}" >"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" fi @@ -177,20 +180,12 @@ EOF if [ "${ACCOUNTS_SSH_AUTHORIZED_KEYS}" != "" ]; then cp -r "${ACCOUNTS_SSH_AUTHORIZED_KEYS}" "${BOOTSTRAP_TMPDIR}/accounts_ssh_authorized_keys" fi - if [ "${NODE_OPERATOR_PRIVATE_KEY}" != "" ]; then - cp "${NODE_OPERATOR_PRIVATE_KEY}" "${BOOTSTRAP_TMPDIR}/node_operator_private_key.pem" - fi - # Create guestos config.json (but not break testing) - if [ -f "/boot/config/config.json" ]; then - /opt/ic/bin/config generate-guestos-config - echo "* Copying 'config-guestos.json' to GuestOS config partition..." - if [ -f "/boot/config/config-guestos.json" ]; then - cp /boot/config/config-guestos.json "${BOOTSTRAP_TMPDIR}/config.json" - # else - # todo: fix ERROR - fi - fi + # Create guestos config.json + echo "* Generating 'config-guestos.json'..." + /opt/ic/bin/config generate-guestos-config + echo "* Copying 'config-guestos.json' to GuestOS config partition..." + cp /boot/config/config-guestos.json "${BOOTSTRAP_TMPDIR}/config.json" tar cf "${OUT_FILE}" \ --sort=name \ From 5758e4b1aebdbf6e7c81f6f98dd078cb69767424 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 18:59:07 +0000 Subject: [PATCH 068/241] Organize build-bootstrap-config-image --- .../build-bootstrap-config-image.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index b287612537e..c24eabc846e 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -151,6 +151,7 @@ function build_ic_bootstrap_tar() { local BOOTSTRAP_TMPDIR=$(mktemp -d) + # todo: delete network.conf and malicious_behaviour.conf cat >"${BOOTSTRAP_TMPDIR}/network.conf" <"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" + fi + + # todo: switch nns_public_key.pem, node_operator_private_key.pem. and accounts_ssh to use config object if [ "${NNS_PUBLIC_KEY}" != "" ]; then cp "${NNS_PUBLIC_KEY}" "${BOOTSTRAP_TMPDIR}/nns_public_key.pem" fi if [ "${NODE_OPERATOR_PRIVATE_KEY}" != "" ]; then cp "${NODE_OPERATOR_PRIVATE_KEY}" "${BOOTSTRAP_TMPDIR}/node_operator_private_key.pem" fi - if [ "${MALICIOUS_BEHAVIOR}" != "" ]; then - echo "malicious_behavior=${MALICIOUS_BEHAVIOR}" >"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" + if [ "${ACCOUNTS_SSH_AUTHORIZED_KEYS}" != "" ]; then + cp -r "${ACCOUNTS_SSH_AUTHORIZED_KEYS}" "${BOOTSTRAP_TMPDIR}/accounts_ssh_authorized_keys" fi + + # todo: investigate what to do for... if [ "${IC_CRYPTO}" != "" ]; then cp -r "${IC_CRYPTO}" "${BOOTSTRAP_TMPDIR}/ic_crypto" fi @@ -177,9 +185,6 @@ EOF if [ "${IC_REGISTRY_LOCAL_STORE}" != "" ]; then cp -r "${IC_REGISTRY_LOCAL_STORE}" "${BOOTSTRAP_TMPDIR}/ic_registry_local_store" fi - if [ "${ACCOUNTS_SSH_AUTHORIZED_KEYS}" != "" ]; then - cp -r "${ACCOUNTS_SSH_AUTHORIZED_KEYS}" "${BOOTSTRAP_TMPDIR}/accounts_ssh_authorized_keys" - fi # Create guestos config.json echo "* Generating 'config-guestos.json'..." From 114b07aa80595c7db953d46d6e602424017d1bfb Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 19:37:02 +0000 Subject: [PATCH 069/241] Remove unnecessary elastic-search argument --- .../generate-guestos-config/dev-generate-guestos-config.sh | 1 - .../generate-guestos-config/generate-guestos-config.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index f5385107ab0..ca3877e3370 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -72,7 +72,6 @@ function read_config_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--nns_public_key "$nns_public_key") - cmd+=(--elasticsearch_hosts "$elasticsearch_hosts") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 1f95d4e3e9f..662f1873810 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -71,7 +71,6 @@ function read_config_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--nns_public_key "$nns_public_key") - cmd+=(--elasticsearch_hosts "$elasticsearch_hosts") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then From beacf9f64a31475a166592d648facdc3172a46b0 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 19:38:10 +0000 Subject: [PATCH 070/241] Remove unnecessary nns_url parameter --- .../generate-guestos-config/dev-generate-guestos-config.sh | 1 - .../generate-guestos-config/generate-guestos-config.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index ca3877e3370..4b03a87c283 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -80,7 +80,6 @@ function assemble_config_media() { cmd+=(--domain "${domain}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") - cmd+=(--nns_url "$nns_urls") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 662f1873810..b4ec771bd8f 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -79,7 +79,6 @@ function assemble_config_media() { cmd+=(--domain "${domain}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") - cmd+=(--nns_url "$nns_urls") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi From 4c7c35d430e54cea7ece43e072774186b49fbb15 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 14 Oct 2024 19:48:25 +0000 Subject: [PATCH 071/241] Reorganize build-bootstrap-config-image parameters --- .../generate-guestos-config/dev-generate-guestos-config.sh | 2 +- .../generate-guestos-config/generate-guestos-config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 4b03a87c283..74dac4c9f2b 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -71,7 +71,6 @@ function read_config_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) - cmd+=(--nns_public_key "$nns_public_key") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then @@ -80,6 +79,7 @@ function assemble_config_media() { cmd+=(--domain "${domain}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + cmd+=(--nns_public_key "$nns_public_key") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index b4ec771bd8f..c2ac0f3239a 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -70,7 +70,6 @@ function read_config_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) - cmd+=(--nns_public_key "$nns_public_key") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then @@ -79,6 +78,7 @@ function assemble_config_media() { cmd+=(--domain "${domain}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + cmd+=(--nns_public_key "$nns_public_key") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") fi From 30e8f3ca7e341a3548167025ee613edd95b46fe3 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 15 Oct 2024 17:59:11 +0000 Subject: [PATCH 072/241] Update filebeat comment --- ic-os/components/selinux/filebeat/filebeat.te | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/selinux/filebeat/filebeat.te b/ic-os/components/selinux/filebeat/filebeat.te index 2a456d9509b..5cf93e4a3e2 100644 --- a/ic-os/components/selinux/filebeat/filebeat.te +++ b/ic-os/components/selinux/filebeat/filebeat.te @@ -20,7 +20,7 @@ type filebeat_var_log_t; files_type(filebeat_var_log_t) # The run-time generated configuration file (and its parent directory): -# /run/ic-node/etc/filebeat{/filebeat.conf) +# /run/ic-node/etc/filebeat type filebeat_conf_t; files_type(filebeat_conf_t) From b9abed9a901eefd8fdec9ae8e026fbde4739e196 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 15 Oct 2024 21:42:32 +0000 Subject: [PATCH 073/241] Add note on config.sh return values --- ic-os/components/misc/config/config.sh | 3 +++ ic-os/components/misc/config/setupos/config.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ic-os/components/misc/config/config.sh b/ic-os/components/misc/config/config.sh index 831d04cbc28..b6ce647f1a6 100644 --- a/ic-os/components/misc/config/config.sh +++ b/ic-os/components/misc/config/config.sh @@ -5,6 +5,9 @@ # Retrieves a value from the config.json file using a JSON path. # Arguments: # $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') +# Note: +# - If the key is not found, this function will produce an empty string. +# - If the value at the key is `null` (e.g., if the Rust type is an `Option` with value `None`), it will output 'null' as a string. function get_config_value() { local CONFIG_FILE="/boot/config/config.json" local key=$1 diff --git a/ic-os/components/misc/config/setupos/config.sh b/ic-os/components/misc/config/setupos/config.sh index 21151813cc8..8b220efc4c6 100644 --- a/ic-os/components/misc/config/setupos/config.sh +++ b/ic-os/components/misc/config/setupos/config.sh @@ -5,6 +5,9 @@ # Retrieves a value from the config.json file using a JSON path. # Arguments: # $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') +# Note: +# - If the key is not found, this function will produce an empty string. +# - If the value at the key is `null` (e.g., if the Rust type is an `Option` with value `None`), it will output 'null' as a string. function get_config_value() { local CONFIG_FILE="/var/ic/config/config.json" local key=$1 From 8ebb285ee011f772ae6f988259ada88e657c16eb Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 15:53:52 +0000 Subject: [PATCH 074/241] Fix ic-btc-mainnet-adapter.service --- .../components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service b/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service index dd591d6c5fc..d90aed38d54 100644 --- a/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service +++ b/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service @@ -11,7 +11,7 @@ StartLimitIntervalSec=0 [Service] User=ic-replica -ExecStartPre=+/opt/ic/bin/generate-btc-adapter-config.sh -o /run/ic-node/config/ic-btc-mainnet-adapter.json5 +ExecStartPre=+/opt/ic/bin/generate-btc-adapter-config.sh -m -o /run/ic-node/config/ic-btc-mainnet-adapter.json5 ExecStart=/opt/ic/bin/ic-btc-adapter /run/ic-node/config/ic-btc-mainnet-adapter.json5 NotifyAccess=main Restart=always From ff8f911eb26d73c35266ba9c0a06c49d83ec7386 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 15:56:37 +0000 Subject: [PATCH 075/241] Fix generate-filebeat-config.sh --- .../components/monitoring/filebeat/generate-filebeat-config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh index 99df7491fea..a8ed5b9889c 100755 --- a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh +++ b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh @@ -47,7 +47,7 @@ read_config_variables ELASTICSEARCH_HOSTS="${elasticsearch_hosts}" ELASTICSEARCH_TAGS="${elasticsearch_tags}" -if [ "${ELASTICSEARCH_HOSTS}" != "" && ["${ELASTICSEARCH_HOSTS}" != "null"]; then +if [ "${ELASTICSEARCH_HOSTS}" != "" ] && ["${ELASTICSEARCH_HOSTS}" != "null"]; then # Covert string into comma separated array if [ "$(echo ${ELASTICSEARCH_HOSTS} | grep ':')" ]; then elasticsearch_hosts_array=$(for host in ${ELASTICSEARCH_HOSTS}; do echo -n "\"${host}\", "; done | sed -E "s@, \$@@g") From 4226469c4277e48db9f16cc4a349274823957b7a Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 16:23:55 +0000 Subject: [PATCH 076/241] Fix generate-filebeat-config.sh --- .../monitoring/filebeat/generate-filebeat-config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh index a8ed5b9889c..2287a7e3e3a 100755 --- a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh +++ b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh @@ -47,7 +47,7 @@ read_config_variables ELASTICSEARCH_HOSTS="${elasticsearch_hosts}" ELASTICSEARCH_TAGS="${elasticsearch_tags}" -if [ "${ELASTICSEARCH_HOSTS}" != "" ] && ["${ELASTICSEARCH_HOSTS}" != "null"]; then +if [ "${ELASTICSEARCH_HOSTS}" != "" ] && [ "${ELASTICSEARCH_HOSTS}" != "null"]; then # Covert string into comma separated array if [ "$(echo ${ELASTICSEARCH_HOSTS} | grep ':')" ]; then elasticsearch_hosts_array=$(for host in ${ELASTICSEARCH_HOSTS}; do echo -n "\"${host}\", "; done | sed -E "s@, \$@@g") @@ -57,7 +57,7 @@ if [ "${ELASTICSEARCH_HOSTS}" != "" ] && ["${ELASTICSEARCH_HOSTS}" != "null"]; t sed -e "s@{{ elasticsearch_hosts }}@${elasticsearch_hosts_array}@" "${IN_FILE}" >"${OUT_FILE}" fi -if [ "${ELASTICSEARCH_TAGS}" != "" ] && ["${ELASTICSEARCH_TAGS}" != "null"]; then +if [ "${ELASTICSEARCH_TAGS}" != "" ] && [ "${ELASTICSEARCH_TAGS}" != "null"]; then # Covert string into comma separated array elasticsearch_tags_array=$(for tag in ${ELASTICSEARCH_TAGS}; do echo -n "\"${tag}\", "; done | sed -E "s@, \$@@g") sed -e "s@#{{ elasticsearch_tags }}@tags: [${elasticsearch_tags_array}]@" -i "${OUT_FILE}" From 4050a6eecca82abc8bee97313d5b4130cfad7295 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 21:20:56 +0000 Subject: [PATCH 077/241] Update generate-replica-config.sh to not use network.conf --- .../build-bootstrap-config-image.sh | 6 +- .../components/ic/generate-replica-config.sh | 86 +++++++++---------- rs/ic_os/config/src/main.rs | 24 +++++- 3 files changed, 68 insertions(+), 48 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index c20bc764559..2e7b793a9df 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -184,7 +184,11 @@ EOF # Create guestos config.json echo "* Generating 'config-guestos.json'..." - /opt/ic/bin/config generate-guestos-config + if [[ -n "$IPV6_ADDRESS" ]]; then + /opt/ic/bin/config generate-guestos-config --guestos_ipv6_address "$IPV6_ADDRESS" + else + /opt/ic/bin/config generate-guestos-config + fi echo "* Copying 'config-guestos.json' to GuestOS config partition..." cp /boot/config/config-guestos.json "${BOOTSTRAP_TMPDIR}/config.json" diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index ba129d954dc..5ae6d9e25b1 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -12,7 +12,6 @@ Usage: Generate replica config from template file. - -n network.conf: Optional, network configuration description file -m malicious_behavior.conf: Optional, malicious behavior parameters -i infile: input ic.json5.template file @@ -58,11 +57,6 @@ function get_if_address_retries() { } function read_config_variables() { - ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') - ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') - ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') - ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') - domain=$(get_config_value '.network_settings.ipv4_config.domain') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') hostname=$(get_config_value '.icos_settings.hostname') backup_retention_time_secs=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_retention_time_seconds') @@ -71,27 +65,45 @@ function read_config_variables() { query_stats_epoch_length=$(get_config_value '.guestos_settings.guestos_dev_settings.query_stats_epoch_length') # todo: - # "ipv6_address") ipv6_address="${value}" ;; # "malicious_behavior") malicious_behavior="${value}" ;; } -# XXX: the following function is duplicate with generate-network-config.sh -# -- consolidate -# -# Read the network config variables from file. The file must be of the form -# "key=value" for each line with a specific set of keys permissible (see -# code below). -# -# Arguments: -# - $1: Name of the file to be read. -function read_network_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_address") ipv6_address="${value}" ;; - esac - done <"$1" +function configure_ipv6() { + ipv6_config_type=$(get_config_value '.ipv6_config | keys[]') + case "$ipv6_config_type" in + "Deterministic") + echo "GuestOS IPv6 configuration should not be 'Deterministic'." + exit 1 + ;; + "Fixed") + IPV6_ADDRESS=$(get_config_value '.network_settings.ipv6_config.Fixed.address') + ;; + "RouterAdvertisement") + interface=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';')) + IPV6_ADDRESS="$(get_if_address_retries 6 ${interface} 12)" + ;; + *) + echo "ERROR: Unknown IPv6 configuration type." + exit 1 + ;; + esac + + if [ "${IPV6_ADDRESS}" == "" ]; then + echo "Cannot determine an IPv6 address, aborting" + exit 1 + fi +} + +function configure_ipv4() { + IPV4_ADDRESS="" IPV4_GATEWAY="" DOMAIN="" + ipv4_config_present=$(get_config_value '.network_settings.ipv4_config != null') + if [ "$ipv4_config_present" = "true" ]; then + ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') + IPV4_ADDRESS="${ipv4_address}/${ipv4_prefix_length}" + IPV4_GATEWAY=$(get_config_value '.network_settings.ipv4_config.gateway') + DOMAIN=$(get_config_value '.network_settings.ipv4_config.domain') + fi } # Read malicious behavior config variables from file. The file must be of the @@ -110,11 +122,8 @@ function read_malicious_behavior_variables() { done <"$1" } -while getopts "m:n:c:i:o:" OPT; do +while getopts "m:i:o:" OPT; do case "${OPT}" in - n) - NETWORK_CONFIG_FILE="${OPTARG}" - ;; m) MALICIOUS_BEHAVIOR_CONFIG_FILE="${OPTARG}" ;; @@ -136,25 +145,15 @@ if [ "${IN_FILE}" == "" -o "${OUT_FILE}" == "" ]; then exit 1 fi -if [ "${NETWORK_CONFIG_FILE}" != "" -a -e "${NETWORK_CONFIG_FILE}" ]; then - read_network_variables "${NETWORK_CONFIG_FILE}" -fi +configure_ipv6 +configure_ipv4 + if [ "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" != "" -a -e "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" ]; then read_malicious_behavior_variables "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" fi read_config_variables -INTERFACE=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';')) -IPV6_ADDRESS="${ipv6_address%/*}" -IPV6_ADDRESS="${IPV6_ADDRESS:-$(get_if_address_retries 6 ${INTERFACE} 12)}" -if [[ -n "$ipv4_address" && "$ipv4_address" != "null" && -n "$ipv4_prefix_length" && "$ipv4_prefix_length" != "null" ]]; then - IPV4_ADDRESS="${ipv4_address}/${ipv4_prefix_length}" -else - IPV4_ADDRESS="" -fi -IPV4_GATEWAY="${ipv4_gateway:-}" -DOMAIN="${domain:-}" NNS_URLS="${nns_urls:-http://[::1]:8080}" NODE_INDEX="${node_index:-0}" # Default value is 24h @@ -172,11 +171,6 @@ QUERY_STATS_EPOCH_LENGTH="${query_stats_epoch_length:-600}" JAEGER_ADDR="${jaeger_addr:-}" [ "${jaeger_addr}" = "null" ] && JAEGER_ADDR="" -if [ "${IPV6_ADDRESS}" == "" ]; then - echo "Cannot determine an IPv6 address, aborting" - exit 1 -fi - sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \ -e "s@{{ ipv4_gateway }}@${IPV4_GATEWAY}@" \ diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 0b48068e308..0ea1eb8e19c 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -4,6 +4,7 @@ use config::config_ini::{get_config_ini_settings, ConfigIniSettings}; use config::deployment_json::get_deployment_settings; use config::serialize_and_write_config; use std::fs::File; +use std::net::Ipv6Addr; use std::path::{Path, PathBuf}; use config::types::*; @@ -43,6 +44,8 @@ pub enum Commands { hostos_config_json_path: PathBuf, #[arg(long, default_value = config::DEFAULT_HOSTOS_GUESTOS_CONFIG_OBJECT_PATH, value_name = "config-guestos.json")] guestos_config_json_path: PathBuf, + #[arg(long, value_name = "ipv6_address")] + guestos_ipv6_address: Option, }, } @@ -205,6 +208,7 @@ pub fn main() -> Result<()> { Some(Commands::GenerateGuestosConfig { hostos_config_json_path, guestos_config_json_path, + guestos_ipv6_address, }) => { let hostos_config_json_path = Path::new(&hostos_config_json_path); @@ -218,8 +222,26 @@ pub fn main() -> Result<()> { *path = guestos_config_path.join("accounts_ssh_authorized_keys"); } + let mut guestos_network_settings = hostos_config.network_settings; + // Update the GuestOS networking if `guestos_ipv6_address` is provided + if let Some(guestos_ipv6_address) = guestos_ipv6_address { + match &guestos_network_settings.ipv6_config { + Ipv6Config::Deterministic(deterministic_ipv6_config) => { + guestos_network_settings.ipv6_config = Ipv6Config::Fixed(FixedIpv6Config { + address: guestos_ipv6_address, + gateway: deterministic_ipv6_config.gateway, + }); + } + _ => { + anyhow::bail!( + "HostOSConfig Ipv6Config should always be of type Deterministic. Cannot reassign GuestOS networking." + ); + } + } + } + let guestos_config = GuestOSConfig { - network_settings: hostos_config.network_settings, + network_settings: guestos_network_settings, icos_settings: guestos_icos_settings, guestos_settings: hostos_config.guestos_settings, }; From c4104f3fa7a7e03c6e3e8c032b91caa11ebcd870 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 22:48:29 +0000 Subject: [PATCH 078/241] Fix guestos-ipv6-address agument --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 2e7b793a9df..1ac4531268d 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -185,7 +185,7 @@ EOF # Create guestos config.json echo "* Generating 'config-guestos.json'..." if [[ -n "$IPV6_ADDRESS" ]]; then - /opt/ic/bin/config generate-guestos-config --guestos_ipv6_address "$IPV6_ADDRESS" + /opt/ic/bin/config generate-guestos-config --guestos-ipv6-address "$IPV6_ADDRESS" else /opt/ic/bin/config generate-guestos-config fi From b9e508e86a759611972891230f4a921d9a626391 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 22:57:44 +0000 Subject: [PATCH 079/241] Update setup-hostname.sh to use config object --- .../setup-hostname/setup-hostname.sh | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh index 1d794bace26..dcbf5558fe8 100755 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh @@ -2,29 +2,16 @@ set -ex -# Read the network config variables from file. The file must be of the form -# "key=value" for each line with a specific set of keys permissible (see -# code below). -# -# Arguments: -# - $1: Name of the file to be read. -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "hostname") hostname="${value}" ;; - esac - done <"$1" +source /opt/ic/bin/config.sh + +function read_config_variables() { + hostname=$(get_config_value '.icos_settings.hostname') + if [[ -z "${hostname}" || "${hostname}" == "null" ]]; then + hostname="unnamed" + fi } -if [ -e /boot/config/network.conf ]; then - cat /boot/config/network.conf - read_variables /boot/config/network.conf - hostname="${hostname:-blank}" -else - hostname="unnamed" -fi +read_config_variables echo "${hostname}" >/run/ic-node/etc/hostname mount --bind /run/ic-node/etc/hostname /etc/hostname From b68e457c132ad3f74c0af93e13c3377a327310bc Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 23:19:02 +0000 Subject: [PATCH 080/241] Convert fixed ipv6 address to a String --- .../build-bootstrap-config-image.sh | 6 +---- rs/ic_os/config/src/main.rs | 27 +++++++++---------- rs/ic_os/config/src/types.rs | 3 ++- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 1ac4531268d..417c460d2f9 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -184,11 +184,7 @@ EOF # Create guestos config.json echo "* Generating 'config-guestos.json'..." - if [[ -n "$IPV6_ADDRESS" ]]; then - /opt/ic/bin/config generate-guestos-config --guestos-ipv6-address "$IPV6_ADDRESS" - else - /opt/ic/bin/config generate-guestos-config - fi + /opt/ic/bin/config generate-guestos-config --guestos-ipv6-address "$IPV6_ADDRESS" echo "* Copying 'config-guestos.json' to GuestOS config partition..." cp /boot/config/config-guestos.json "${BOOTSTRAP_TMPDIR}/config.json" diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 0ea1eb8e19c..83b928ec621 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -4,7 +4,6 @@ use config::config_ini::{get_config_ini_settings, ConfigIniSettings}; use config::deployment_json::get_deployment_settings; use config::serialize_and_write_config; use std::fs::File; -use std::net::Ipv6Addr; use std::path::{Path, PathBuf}; use config::types::*; @@ -45,7 +44,7 @@ pub enum Commands { #[arg(long, default_value = config::DEFAULT_HOSTOS_GUESTOS_CONFIG_OBJECT_PATH, value_name = "config-guestos.json")] guestos_config_json_path: PathBuf, #[arg(long, value_name = "ipv6_address")] - guestos_ipv6_address: Option, + guestos_ipv6_address: String, }, } @@ -224,19 +223,17 @@ pub fn main() -> Result<()> { let mut guestos_network_settings = hostos_config.network_settings; // Update the GuestOS networking if `guestos_ipv6_address` is provided - if let Some(guestos_ipv6_address) = guestos_ipv6_address { - match &guestos_network_settings.ipv6_config { - Ipv6Config::Deterministic(deterministic_ipv6_config) => { - guestos_network_settings.ipv6_config = Ipv6Config::Fixed(FixedIpv6Config { - address: guestos_ipv6_address, - gateway: deterministic_ipv6_config.gateway, - }); - } - _ => { - anyhow::bail!( - "HostOSConfig Ipv6Config should always be of type Deterministic. Cannot reassign GuestOS networking." - ); - } + match &guestos_network_settings.ipv6_config { + Ipv6Config::Deterministic(deterministic_ipv6_config) => { + guestos_network_settings.ipv6_config = Ipv6Config::Fixed(FixedIpv6Config { + address: guestos_ipv6_address, + gateway: deterministic_ipv6_config.gateway, + }); + } + _ => { + anyhow::bail!( + "HostOSConfig Ipv6Config should always be of type Deterministic. Cannot reassign GuestOS networking." + ); } } diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 1ca9963492a..d7b3b586aff 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -145,6 +145,7 @@ pub struct DeterministicIpv6Config { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct FixedIpv6Config { - pub address: Ipv6Addr, + // fixed ipv6 address includes subnet mask /64 + pub address: String, pub gateway: Ipv6Addr, } From f237d01cec34ccc3d8f5a1edd0bc54b2c1b354c2 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 23:21:45 +0000 Subject: [PATCH 081/241] Fix config.sh comment --- ic-os/components/misc/config/config.sh | 4 +--- ic-os/components/misc/config/setupos/config.sh | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ic-os/components/misc/config/config.sh b/ic-os/components/misc/config/config.sh index b6ce647f1a6..7cd800317d8 100644 --- a/ic-os/components/misc/config/config.sh +++ b/ic-os/components/misc/config/config.sh @@ -5,9 +5,7 @@ # Retrieves a value from the config.json file using a JSON path. # Arguments: # $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') -# Note: -# - If the key is not found, this function will produce an empty string. -# - If the value at the key is `null` (e.g., if the Rust type is an `Option` with value `None`), it will output 'null' as a string. +# Note: If the key is not found, this function will return null. function get_config_value() { local CONFIG_FILE="/boot/config/config.json" local key=$1 diff --git a/ic-os/components/misc/config/setupos/config.sh b/ic-os/components/misc/config/setupos/config.sh index 8b220efc4c6..57198454f99 100644 --- a/ic-os/components/misc/config/setupos/config.sh +++ b/ic-os/components/misc/config/setupos/config.sh @@ -5,9 +5,7 @@ # Retrieves a value from the config.json file using a JSON path. # Arguments: # $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') -# Note: -# - If the key is not found, this function will produce an empty string. -# - If the value at the key is `null` (e.g., if the Rust type is an `Option` with value `None`), it will output 'null' as a string. +# Note: If the key is not found, this function will return null. function get_config_value() { local CONFIG_FILE="/var/ic/config/config.json" local key=$1 From bfe38ac1b8355541f8e7fe9ec41bcc01cd1b3721 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 23:30:53 +0000 Subject: [PATCH 082/241] Refactor generate-replica-config --- .../components/ic/generate-replica-config.sh | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 5ae6d9e25b1..27e27fca952 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -57,12 +57,12 @@ function get_if_address_retries() { } function read_config_variables() { - nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') - hostname=$(get_config_value '.icos_settings.hostname') - backup_retention_time_secs=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_retention_time_seconds') - backup_purging_interval_secs=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_purging_interval_seconds') - jaeger_addr=$(get_config_value '.guestos_settings.guestos_dev_settings.jaeger_addr') - query_stats_epoch_length=$(get_config_value '.guestos_settings.guestos_dev_settings.query_stats_epoch_length') + NNS_URLS=$(get_config_value '.icos_settings.nns_urls | join(",")') + NODE_INDEX=$(get_config_value '.icos_settings.hostname') + BACKUP_RETENTION_TIME_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_retention_time_seconds') + BACKUP_PURGING_INTERVAL_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_purging_interval_seconds') + QUERY_STATS_EPOCH_LENGTH=$(get_config_value '.guestos_settings.guestos_dev_settings.query_stats_epoch_length') + JAEGER_ADDR=$(get_config_value '.guestos_settings.guestos_dev_settings.jaeger_addr') # todo: # "malicious_behavior") malicious_behavior="${value}" ;; @@ -154,22 +154,16 @@ fi read_config_variables -NNS_URLS="${nns_urls:-http://[::1]:8080}" -NODE_INDEX="${node_index:-0}" +[ "${NNS_URLS}" = "null" ] && NNS_URLS="http://[::1]:8080" +[ "${NODE_INDEX}" = "null" ] && NODE_INDEX="0" # Default value is 24h -BACKUP_RETENTION_TIME_SECS="${backup_retention_time_secs:-86400}" -[ "${backup_retention_time_secs}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" +[ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default value is 1h -BACKUP_PURGING_INTERVAL_SECS="${backup_purging_interval_secs:-3600}" -[ "${backup_purging_interval_secs}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" -# Default is null (None) -MALICIOUS_BEHAVIOR="${malicious_behavior:-null}" +[ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default is 600 blocks i.e. around 10min -QUERY_STATS_EPOCH_LENGTH="${query_stats_epoch_length:-600}" -[ "${query_stats_epoch_length}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" +[ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # TODO: If the Jaeger address is not specified the config file will contain Some(""). This needs to be fixed. -JAEGER_ADDR="${jaeger_addr:-}" -[ "${jaeger_addr}" = "null" ] && JAEGER_ADDR="" +[ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \ From 30353a45e67867481c7a6cf62d3e78b00de63c52 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 23:32:03 +0000 Subject: [PATCH 083/241] Reorganize function ordering --- .../components/ic/generate-replica-config.sh | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 27e27fca952..43ba153fea3 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -19,43 +19,6 @@ Usage: EOF } -# Get address of interface -# -# Arguments: -# - $1: address family (4 or 6 for IPv4 or IPv6) -# - $2: interface name -function get_if_address() { - local FAMILY=-"$1" - local INTERFACE="$2" - ip -o "${FAMILY}" addr show up primary scope global "${INTERFACE}" | while read -r num dev family addr options; do - echo ${addr%/*} - break - done -} - -# Get address of interface, retrying for a while -# -# Arguments: -# - $1: address family (4 or 6 for IPv4 or IPv6) -# - $2: interface name -# - $3: number of retries, trying every second -function get_if_address_retries() { - local FAMILY=-"$1" - local INTERFACE="$2" - local RETRIES="$3" - local ADDR="" - while [ "${RETRIES}" != 0 -a "$ADDR" == "" ]; do - ADDR=$(get_if_address "${FAMILY}" "${INTERFACE}") - if [ "${ADDR}" != "" ]; then - echo "${ADDR}" - break - fi - RETRIES=$(("${RETRIES}" - 1)) - echo "Retrying ${RETRIES} ..." 1>&2 - sleep 10 - done -} - function read_config_variables() { NNS_URLS=$(get_config_value '.icos_settings.nns_urls | join(",")') NODE_INDEX=$(get_config_value '.icos_settings.hostname') @@ -106,6 +69,43 @@ function configure_ipv4() { fi } +# Get address of interface +# +# Arguments: +# - $1: address family (4 or 6 for IPv4 or IPv6) +# - $2: interface name +function get_if_address() { + local FAMILY=-"$1" + local INTERFACE="$2" + ip -o "${FAMILY}" addr show up primary scope global "${INTERFACE}" | while read -r num dev family addr options; do + echo ${addr%/*} + break + done +} + +# Get address of interface, retrying for a while +# +# Arguments: +# - $1: address family (4 or 6 for IPv4 or IPv6) +# - $2: interface name +# - $3: number of retries, trying every second +function get_if_address_retries() { + local FAMILY=-"$1" + local INTERFACE="$2" + local RETRIES="$3" + local ADDR="" + while [ "${RETRIES}" != 0 -a "$ADDR" == "" ]; do + ADDR=$(get_if_address "${FAMILY}" "${INTERFACE}") + if [ "${ADDR}" != "" ]; then + echo "${ADDR}" + break + fi + RETRIES=$(("${RETRIES}" - 1)) + echo "Retrying ${RETRIES} ..." 1>&2 + sleep 10 + done +} + # Read malicious behavior config variables from file. The file must be of the # form "key=value" for each line with a specific set of keys permissible (see # code below). @@ -154,6 +154,7 @@ fi read_config_variables + [ "${NNS_URLS}" = "null" ] && NNS_URLS="http://[::1]:8080" [ "${NODE_INDEX}" = "null" ] && NODE_INDEX="0" # Default value is 24h From 542537cf184de5eff7e860c2fb0f0725276658de Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 23:35:09 +0000 Subject: [PATCH 084/241] Create set_default_config_values --- .../components/ic/generate-replica-config.sh | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 43ba153fea3..adf446e6e00 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -106,6 +106,17 @@ function get_if_address_retries() { done } +function set_default_config_values() { + [ "${NNS_URLS}" = "null" ] && NNS_URLS="http://[::1]:8080" + [ "${NODE_INDEX}" = "null" ] && NODE_INDEX="0" + [ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default value is 24h + [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default value is 1h + [ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # Default is 600 blocks (around 10min) + + # TODO: If the Jaeger address is not specified the config file will contain Some(""). This needs to be fixed. + [ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" +} + # Read malicious behavior config variables from file. The file must be of the # form "key=value" for each line with a specific set of keys permissible (see # code below). @@ -153,18 +164,7 @@ if [ "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" != "" -a -e "${MALICIOUS_BEHAVIOR_CONFI fi read_config_variables - - -[ "${NNS_URLS}" = "null" ] && NNS_URLS="http://[::1]:8080" -[ "${NODE_INDEX}" = "null" ] && NODE_INDEX="0" -# Default value is 24h -[ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" -# Default value is 1h -[ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" -# Default is 600 blocks i.e. around 10min -[ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" -# TODO: If the Jaeger address is not specified the config file will contain Some(""). This needs to be fixed. -[ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" +set_default_config_values sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \ From 6ac4aae71774b209dbfa251ab6c5f780fede71c9 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 23:54:24 +0000 Subject: [PATCH 085/241] Fix generate-filebeat-config --- .../monitoring/filebeat/generate-filebeat-config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh index 2287a7e3e3a..1b3879f9d12 100755 --- a/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh +++ b/ic-os/components/monitoring/filebeat/generate-filebeat-config.sh @@ -47,7 +47,7 @@ read_config_variables ELASTICSEARCH_HOSTS="${elasticsearch_hosts}" ELASTICSEARCH_TAGS="${elasticsearch_tags}" -if [ "${ELASTICSEARCH_HOSTS}" != "" ] && [ "${ELASTICSEARCH_HOSTS}" != "null"]; then +if [ "${ELASTICSEARCH_HOSTS}" != "" ] && [ "${ELASTICSEARCH_HOSTS}" != "null" ]; then # Covert string into comma separated array if [ "$(echo ${ELASTICSEARCH_HOSTS} | grep ':')" ]; then elasticsearch_hosts_array=$(for host in ${ELASTICSEARCH_HOSTS}; do echo -n "\"${host}\", "; done | sed -E "s@, \$@@g") @@ -57,7 +57,7 @@ if [ "${ELASTICSEARCH_HOSTS}" != "" ] && [ "${ELASTICSEARCH_HOSTS}" != "null"]; sed -e "s@{{ elasticsearch_hosts }}@${elasticsearch_hosts_array}@" "${IN_FILE}" >"${OUT_FILE}" fi -if [ "${ELASTICSEARCH_TAGS}" != "" ] && [ "${ELASTICSEARCH_TAGS}" != "null"]; then +if [ "${ELASTICSEARCH_TAGS}" != "" ] && [ "${ELASTICSEARCH_TAGS}" != "null" ]; then # Covert string into comma separated array elasticsearch_tags_array=$(for tag in ${ELASTICSEARCH_TAGS}; do echo -n "\"${tag}\", "; done | sed -E "s@, \$@@g") sed -e "s@#{{ elasticsearch_tags }}@tags: [${elasticsearch_tags_array}]@" -i "${OUT_FILE}" From 94c889aad14e11a02601892c673de12894ea302c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 16 Oct 2024 23:54:43 +0000 Subject: [PATCH 086/241] Fix get_config_value for ipv6_config --- ic-os/components/ic/generate-replica-config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index adf446e6e00..9959b560dc2 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -32,7 +32,7 @@ function read_config_variables() { } function configure_ipv6() { - ipv6_config_type=$(get_config_value '.ipv6_config | keys[]') + ipv6_config_type=$(get_config_value '.network_settings.ipv6_config | keys[]') case "$ipv6_config_type" in "Deterministic") echo "GuestOS IPv6 configuration should not be 'Deterministic'." From b24be332ca630f106cdc3ae82732c8f2e63de46a Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 15:16:49 +0000 Subject: [PATCH 087/241] Fix node_ID bug --- ic-os/components/ic/generate-replica-config.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 84ee4156f51..feea11b0c93 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -21,7 +21,6 @@ EOF function read_config_variables() { NNS_URLS=$(get_config_value '.icos_settings.nns_urls | join(",")') - NODE_INDEX=$(get_config_value '.icos_settings.hostname') BACKUP_RETENTION_TIME_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_retention_time_seconds') BACKUP_PURGING_INTERVAL_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_purging_interval_seconds') QUERY_STATS_EPOCH_LENGTH=$(get_config_value '.guestos_settings.guestos_dev_settings.query_stats_epoch_length') @@ -108,13 +107,15 @@ function get_if_address_retries() { function set_default_config_values() { [ "${NNS_URLS}" = "null" ] && NNS_URLS="http://[::1]:8080" - [ "${NODE_INDEX}" = "null" ] && NODE_INDEX="0" [ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default value is 24h [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default value is 1h [ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # Default is 600 blocks (around 10min) # TODO: If the Jaeger address is not specified the config file will contain Some(""). This needs to be fixed. [ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" + + # todo: remove node_index variable and hard-code into ic.json5.template + NODE_INDEX="0" } # Read malicious behavior config variables from file. The file must be of the From 06b9daa8f4733936218173952c8b82db2439035e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 15:27:35 +0000 Subject: [PATCH 088/241] Remove network.conf from guestos config --- .../build-bootstrap-config-image.sh | 46 +------------------ .../dev-generate-guestos-config.sh | 12 ----- .../generate-guestos-config.sh | 12 ----- ic-os/components/ic/ic-replica.service | 2 +- 4 files changed, 2 insertions(+), 70 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 417c460d2f9..8264328e962 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -20,22 +20,6 @@ options may be specified: The IPv6 address to assign. Must include netmask in bits (e.g. dead:beef::1/64). Overrides all other generation for testing. - --ipv6_gateway a:b::c - Default IPv6 gateway. - - --ipv4_address a.b.c.d/n - (optional) The IPv4 address to assign. Must include prefix length (e.g. - 18.208.190.35/28). - - --ipv4_gateway a.b.c.d - (optional) Default IPv4 gateway (e.g. 18.208.190.33). - - --domain domain - (optional) The domain name to assign to the guest. - - --hostname name - Name to assign to the host. Will be used in logging. - --ic_crypto path Injected crypto state. Should point to a directory containing material generated by ic-prep. Typically, this is IC_PREP_OUT_PATH/node-X/crypto. @@ -79,7 +63,7 @@ function build_ic_bootstrap_tar() { local OUT_FILE="$1" shift - local IPV6_ADDRESS IPV6_GATEWAY DOMAIN HOSTNAME + local IPV6_ADDRESS local IC_CRYPTO IC_STATE IC_REGISTRY_LOCAL_STORE local NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY local ACCOUNTS_SSH_AUTHORIZED_KEYS @@ -94,21 +78,6 @@ function build_ic_bootstrap_tar() { --ipv6_address) IPV6_ADDRESS="$2" ;; - --ipv6_gateway) - IPV6_GATEWAY="$2" - ;; - --ipv4_address) - IPV4_ADDRESS="$2" - ;; - --ipv4_gateway) - IPV4_GATEWAY="$2" - ;; - --domain) - DOMAIN="$2" - ;; - --hostname) - HOSTNAME="$2" - ;; --ic_crypto) IC_CRYPTO="$2" ;; @@ -140,22 +109,9 @@ function build_ic_bootstrap_tar() { shift 2 done - [[ "$HOSTNAME" == "" ]] || [[ "$HOSTNAME" =~ [a-zA-Z]*([a-zA-Z0-9])*(-+([a-zA-Z0-9])) ]] || { - echo "Invalid hostname: '$HOSTNAME'" >&2 - exit 1 - } - local BOOTSTRAP_TMPDIR=$(mktemp -d) # todo: delete network.conf and malicious_behaviour.conf - cat >"${BOOTSTRAP_TMPDIR}/network.conf" <"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" fi diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 74dac4c9f2b..449c1be27b8 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -55,11 +55,6 @@ OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') - ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') - ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') - ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') - ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') - domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') @@ -72,13 +67,6 @@ function read_config_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") - cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then - cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") - cmd+=(--ipv4_gateway "${ipv4_gateway}") - cmd+=(--domain "${domain}") - fi - cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_public_key "$nns_public_key") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index c2ac0f3239a..2e2a9c581f4 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -55,11 +55,6 @@ OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') - ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') - ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') - ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') - ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') - domain=$(get_config_value '.network_settings.ipv4_config.domain') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') @@ -71,13 +66,6 @@ function read_config_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") - cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then - cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") - cmd+=(--ipv4_gateway "${ipv4_gateway}") - cmd+=(--domain "${domain}") - fi - cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_public_key "$nns_public_key") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") diff --git a/ic-os/components/ic/ic-replica.service b/ic-os/components/ic/ic-replica.service index 4680399641a..293a0b88625 100644 --- a/ic-os/components/ic/ic-replica.service +++ b/ic-os/components/ic/ic-replica.service @@ -28,7 +28,7 @@ Environment=RUST_BACKTRACE=1 # Remember to update 'rs/default.nix' for nix-shell users # Remember to update 'src/dfx/src/actors/replica.rs' in the sdk repo for dfx users Environment=RUST_MIN_STACK=8192000 -ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -m /boot/config/malicious_behavior.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 +ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -m /boot/config/malicious_behavior.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 ExecStart=/opt/ic/bin/orchestrator --replica-binary-dir /var/lib/ic/data/images --cup-dir /var/lib/ic/data/cups --replica-config-file /run/ic-node/config/ic.json5 --enable-provisional-registration --ic-binary-directory /opt/ic/bin --orchestrator-data-directory /var/lib/ic/data/orchestrator --version-file /opt/ic/share/version.txt LimitNOFILE=16777216 Restart=always From 4b8da4a3d9e1c81a13fabdf1b0287e0432bb9583 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 19:52:49 +0000 Subject: [PATCH 089/241] Add mgmt_mac to ICOSSettings and rename deployment_environment --- rs/ic_os/config/src/lib.rs | 9 ++- rs/ic_os/config/src/main.rs | 17 ++++- rs/ic_os/config/src/types.rs | 9 +-- rs/ic_os/os_tools/hostos_tool/src/main.rs | 87 ++++------------------ rs/ic_os/os_tools/setupos_tool/src/main.rs | 35 +-------- 5 files changed, 40 insertions(+), 117 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index eebbed0263a..bc79a61a795 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -49,7 +49,7 @@ mod tests { use types::*; #[test] - fn test_serialize_and_deserialize() { + fn test_serialize_and_deserialize() -> Result<(), Box> { let ipv6_config = Ipv6Config::Deterministic(DeterministicIpv6Config { prefix: "2a00:fb01:400:200".to_string(), prefix_length: 64_u8, @@ -69,12 +69,13 @@ mod tests { .join(" "), elasticsearch_tags: None, }; - let icos_dev_settings = ICOSDevSettings { mgmt_mac: None }; + let icos_dev_settings = ICOSDevSettings::default(); let icos_settings = ICOSSettings { + mgmt_mac: FormattedMacAddress::try_from("ec:2a:72:31:a2:0c")?, + deployment_environment: "Mainnet".to_string(), logging, nns_public_key_path: PathBuf::from("/path/to/key"), nns_urls: vec!["http://localhost".parse().unwrap()], - hostname: "mainnet".to_string(), node_operator_private_key_path: None, ssh_authorized_keys_path: None, icos_dev_settings, @@ -131,5 +132,7 @@ mod tests { serialize_and_deserialize(&setupos_config_struct); serialize_and_deserialize(&hostos_config_struct); serialize_and_deserialize(&guestos_config_struct); + + Ok(()) } } diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 83b928ec621..db2cb3b1b65 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -115,22 +115,31 @@ pub fn main() -> Result<()> { elasticsearch_tags: None, }; - let icos_dev_settings = ICOSDevSettings { - mgmt_mac: deployment_json_settings.deployment.mgmt_mac, + let mgmt_mac = match deployment_json_settings.deployment.mgmt_mac { + Some(config_mac) => { + let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; + println!( + "Using mgmt_mac address found in deployment.json: {}", + mgmt_mac + ); + mgmt_mac + } + None => get_ipmi_mac()?, }; let icos_settings = ICOSSettings { + mgmt_mac, + deployment_environment: deployment_json_settings.deployment.name, logging, nns_public_key_path: nns_public_key_path.to_path_buf(), nns_urls: deployment_json_settings.nns.url.clone(), - hostname: deployment_json_settings.deployment.name.to_string(), node_operator_private_key_path: node_operator_private_key_path .exists() .then_some(node_operator_private_key_path), ssh_authorized_keys_path: ssh_authorized_keys_path .exists() .then_some(ssh_authorized_keys_path), - icos_dev_settings, + icos_dev_settings: ICOSDevSettings::default(), }; let setupos_settings = SetupOSSettings; diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index d7b3b586aff..d69ed0b3581 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -83,12 +83,13 @@ pub struct BackupSpoolSettings { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct ICOSSettings { + pub mgmt_mac: FormattedMacAddress, + pub deployment_environment: String, pub logging: Logging, /// This file must be a text file containing the public key of the NNS to be used. pub nns_public_key_path: PathBuf, /// The URL (HTTP) of the NNS node(s). pub nns_urls: Vec, - pub hostname: String, /// This file contains the Node Operator private key, /// which is registered with the NNS and used to sign the IC join request. pub node_operator_private_key_path: Option, @@ -102,10 +103,8 @@ pub struct ICOSSettings { pub icos_dev_settings: ICOSDevSettings, } -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] -pub struct ICOSDevSettings { - pub mgmt_mac: Option, -} +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Default)] +pub struct ICOSDevSettings {} #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct Logging { diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 10711d92a40..163a49abfe1 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -56,26 +56,9 @@ pub fn main() -> Result<()> { &hostos_config.network_settings ); - let mgmt_mac = match hostos_config - .icos_settings - .icos_dev_settings - .mgmt_mac - .as_ref() - { - Some(config_mac) => { - let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; - eprintln!( - "Using mgmt_mac address found in deployment.json: {}", - mgmt_mac - ); - mgmt_mac - } - None => get_ipmi_mac()?, - }; - let generated_mac = generate_mac_address( - &mgmt_mac, - &hostos_config.icos_settings.hostname, + &hostos_config.icos_settings.mgmt_mac, + &hostos_config.icos_settings.deployment_environment, &NodeType::HostOS, )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); @@ -95,26 +78,12 @@ pub fn main() -> Result<()> { &hostos_config.network_settings ); - let mgmt_mac = match hostos_config - .icos_settings - .icos_dev_settings - .mgmt_mac - .as_ref() - { - Some(config_mac) => { - let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; - eprintln!( - "Using mgmt_mac address found in deployment.json: {}", - mgmt_mac - ); - mgmt_mac - } - None => get_ipmi_mac()?, - }; let node_type = node_type.parse::()?; - - let generated_mac = - generate_mac_address(&mgmt_mac, &hostos_config.icos_settings.hostname, &node_type)?; + let generated_mac = generate_mac_address( + &hostos_config.icos_settings.mgmt_mac, + &hostos_config.icos_settings.deployment_environment, + &node_type, + )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = @@ -141,26 +110,12 @@ pub fn main() -> Result<()> { &hostos_config.network_settings ); - let mgmt_mac = match hostos_config - .icos_settings - .icos_dev_settings - .mgmt_mac - .as_ref() - { - Some(config_mac) => { - let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; - eprintln!( - "Using mgmt_mac address found in deployment.json: {}", - mgmt_mac - ); - mgmt_mac - } - None => get_ipmi_mac()?, - }; let node_type = node_type.parse::()?; - - let generated_mac = - generate_mac_address(&mgmt_mac, &hostos_config.icos_settings.hostname, &node_type)?; + let generated_mac = generate_mac_address( + &hostos_config.icos_settings.mgmt_mac, + &hostos_config.icos_settings.deployment_environment, + &node_type, + )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); let generated_mac = FormattedMacAddress::from(&generated_mac); @@ -172,23 +127,7 @@ pub fn main() -> Result<()> { let hostos_config: HostOSConfig = deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; - let mgmt_mac = match hostos_config - .icos_settings - .icos_dev_settings - .mgmt_mac - .as_ref() - { - Some(config_mac) => { - let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; - eprintln!( - "Using mgmt_mac address found in deployment.json: {}", - mgmt_mac - ); - mgmt_mac - } - None => get_ipmi_mac()?, - }; - println!("{}", mgmt_mac); + println!("{}", hostos_config.icos_settings.mgmt_mac); Ok(()) } None => Err(anyhow!( diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 152c6c48e24..f9b0640d20c 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -60,25 +60,9 @@ pub fn main() -> Result<()> { &setupos_config.network_settings ); - let mgmt_mac = match setupos_config - .icos_settings - .icos_dev_settings - .mgmt_mac - .as_ref() - { - Some(config_mac) => { - let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; - eprintln!( - "Using mgmt_mac address found in deployment.json: {}", - mgmt_mac - ); - mgmt_mac - } - None => get_ipmi_mac()?, - }; let generated_mac = generate_mac_address( - &mgmt_mac, - &setupos_config.icos_settings.hostname, + &setupos_config.icos_settings.mgmt_mac, + &setupos_config.icos_settings.deployment_environment, &NodeType::SetupOS, )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); @@ -99,20 +83,9 @@ pub fn main() -> Result<()> { ); let node_type = node_type.parse::()?; - let mgmt_mac = match setupos_config.icos_settings.icos_dev_settings.mgmt_mac { - Some(config_mac) => { - let mgmt_mac = FormattedMacAddress::try_from(config_mac.as_str())?; - eprintln!( - "Using mgmt_mac address found in deployment.json: {}", - mgmt_mac - ); - mgmt_mac - } - None => get_ipmi_mac()?, - }; let generated_mac = generate_mac_address( - &mgmt_mac, - &setupos_config.icos_settings.hostname, + &setupos_config.icos_settings.mgmt_mac, + &setupos_config.icos_settings.deployment_environment, &node_type, )?; eprintln!("Using generated mac (unformatted) {}", generated_mac); From 9878ae0e5db0610200ff0d622b0d6d6d26957546 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 19:55:10 +0000 Subject: [PATCH 090/241] Create mac_address crate to fix cyclic dependency between network and config --- Cargo.lock | 19 +++++++++-- Cargo.toml | 1 + rs/ic_os/config/BUILD.bazel | 1 + rs/ic_os/config/Cargo.toml | 1 + rs/ic_os/config/src/lib.rs | 1 + rs/ic_os/config/src/main.rs | 1 + rs/ic_os/config/src/types.rs | 1 + rs/ic_os/network/BUILD.bazel | 4 +-- rs/ic_os/network/Cargo.toml | 4 +-- rs/ic_os/network/mac_address/BUILD.bazel | 32 +++++++++++++++++++ rs/ic_os/network/mac_address/Cargo.toml | 12 +++++++ rs/ic_os/network/mac_address/src/lib.rs | 2 ++ .../{ => mac_address}/src/mac_address.rs | 9 +++--- .../{ => mac_address}/src/node_type.rs | 0 rs/ic_os/network/src/ipv6.rs | 2 +- rs/ic_os/network/src/lib.rs | 5 +-- rs/ic_os/network/src/systemd.rs | 2 +- rs/ic_os/os_tools/hostos_tool/BUILD.bazel | 1 + rs/ic_os/os_tools/hostos_tool/Cargo.toml | 3 +- rs/ic_os/os_tools/hostos_tool/src/main.rs | 4 +-- rs/ic_os/os_tools/setupos_tool/BUILD.bazel | 1 + rs/ic_os/os_tools/setupos_tool/Cargo.toml | 3 +- rs/ic_os/os_tools/setupos_tool/src/main.rs | 4 +-- 23 files changed, 88 insertions(+), 25 deletions(-) create mode 100644 rs/ic_os/network/mac_address/BUILD.bazel create mode 100644 rs/ic_os/network/mac_address/Cargo.toml create mode 100644 rs/ic_os/network/mac_address/src/lib.rs rename rs/ic_os/network/{ => mac_address}/src/mac_address.rs (96%) rename rs/ic_os/network/{ => mac_address}/src/node_type.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index b1359c9f36f..3f4c62c09cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2268,6 +2268,7 @@ dependencies = [ "anyhow", "clap 4.5.19", "ic-types", + "mac_address", "once_cell", "regex", "serde", @@ -4788,6 +4789,7 @@ dependencies = [ "anyhow", "clap 4.5.19", "config", + "mac_address", "network", "utils", ] @@ -14907,6 +14909,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mac_address" +version = "1.0.0" +dependencies = [ + "anyhow", + "hex", + "regex", + "serde", + "sha2 0.10.8", + "utils", +] + [[package]] name = "mach2" version = "0.4.2" @@ -15364,14 +15378,12 @@ version = "1.0.0" dependencies = [ "anyhow", "config", - "hex", + "mac_address", "ping", "rayon", - "regex", "serde", "serde_json", "serde_with 1.14.0", - "sha2 0.10.8", "utils", ] @@ -18983,6 +18995,7 @@ dependencies = [ "anyhow", "clap 4.5.19", "config", + "mac_address", "network", "utils", ] diff --git a/Cargo.toml b/Cargo.toml index d3b641b1724..e56c98d8375 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,6 +131,7 @@ members = [ "rs/ic_os/build_tools/inject_files", "rs/ic_os/dev_test_tools/launch-single-vm", "rs/ic_os/network", + "rs/ic_os/network/mac_address", "rs/ic_os/nft_exporter", "rs/ic_os/nss_icos", "rs/ic_os/dev_test_tools/setupos-inject-configuration", diff --git a/rs/ic_os/config/BUILD.bazel b/rs/ic_os/config/BUILD.bazel index 2438720360d..6e2ca170df8 100644 --- a/rs/ic_os/config/BUILD.bazel +++ b/rs/ic_os/config/BUILD.bazel @@ -4,6 +4,7 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. + "//rs/ic_os/network/config", "//rs/ic_os/utils", "//rs/types/types", "@crate_index//:anyhow", diff --git a/rs/ic_os/config/Cargo.toml b/rs/ic_os/config/Cargo.toml index 811e805c465..2c2266ff5a5 100644 --- a/rs/ic_os/config/Cargo.toml +++ b/rs/ic_os/config/Cargo.toml @@ -13,6 +13,7 @@ serde_json = { workspace = true } serde = { workspace = true } serde_with = "1.6.2" regex = { workspace = true } +mac_address = { path = "../network/mac_address" } [dev-dependencies] once_cell = "1.8" diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index bc79a61a795..347ee0f9ab6 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -45,6 +45,7 @@ pub fn deserialize_config Deserialize<'de>>(file_path: &str) -> Resu #[cfg(test)] mod tests { use super::*; + use mac_address::mac_address::FormattedMacAddress; use std::path::PathBuf; use types::*; diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index db2cb3b1b65..ae05330f15b 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -3,6 +3,7 @@ use clap::{Parser, Subcommand}; use config::config_ini::{get_config_ini_settings, ConfigIniSettings}; use config::deployment_json::get_deployment_settings; use config::serialize_and_write_config; +use mac_address::mac_address::{get_ipmi_mac, FormattedMacAddress}; use std::fs::File; use std::path::{Path, PathBuf}; diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index d69ed0b3581..83a2258b1d0 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -1,4 +1,5 @@ use ic_types::malicious_behaviour::MaliciousBehaviour; +use mac_address::mac_address::FormattedMacAddress; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, Ipv6Addr}; use std::path::PathBuf; diff --git a/rs/ic_os/network/BUILD.bazel b/rs/ic_os/network/BUILD.bazel index 64f1f412458..c31a236e9ae 100644 --- a/rs/ic_os/network/BUILD.bazel +++ b/rs/ic_os/network/BUILD.bazel @@ -6,15 +6,13 @@ DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", "//rs/ic_os/utils", + "//rs/ic_os/network/mac_address", "@crate_index//:anyhow", - "@crate_index//:hex", "@crate_index//:ping", "@crate_index//:rayon", - "@crate_index//:regex", "@crate_index//:serde", "@crate_index//:serde_json", "@crate_index//:serde_with", - "@crate_index//:sha2", ] rust_library( diff --git a/rs/ic_os/network/Cargo.toml b/rs/ic_os/network/Cargo.toml index bdedaabc04e..e96c8f5f613 100644 --- a/rs/ic_os/network/Cargo.toml +++ b/rs/ic_os/network/Cargo.toml @@ -6,12 +6,10 @@ edition = "2021" [dependencies] anyhow = { workspace = true } config = { path = "../config" } -hex = { version = "^0.4.3" } +mac_address = { path = "./mac_address" } ping = { version = "^0.5.0" } rayon = { workspace = true } -regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_with = { version = "^1.6.2" } -sha2 = { workspace = true } utils = { path = "../utils" } diff --git a/rs/ic_os/network/mac_address/BUILD.bazel b/rs/ic_os/network/mac_address/BUILD.bazel new file mode 100644 index 00000000000..31de5a1bf9e --- /dev/null +++ b/rs/ic_os/network/mac_address/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +package(default_visibility = ["//rs:ic-os-pkg"]) + +DEPENDENCIES = [ + # Keep sorted. + "//rs/ic_os/utils", + "@crate_index//:anyhow", + "@crate_index//:regex", + "@crate_index//:sha2", + "@crate_index//:hex", + "@crate_index//:serde", +] + +rust_library( + name = "mac_address", + srcs = glob( + ["src/**/*.rs"], + ), + aliases = {}, + crate_name = "mac_address", + edition = "2021", + proc_macro_deps = [], + deps = DEPENDENCIES, +) + +rust_test( + name = "test", + size = "small", + crate = ":mac_address", + deps = DEPENDENCIES, +) diff --git a/rs/ic_os/network/mac_address/Cargo.toml b/rs/ic_os/network/mac_address/Cargo.toml new file mode 100644 index 00000000000..6a571cbbe93 --- /dev/null +++ b/rs/ic_os/network/mac_address/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "mac_address" +version = "1.0.0" +edition = "2021" + +[dependencies] +anyhow = { workspace = true } +regex = { workspace = true } +sha2 = { workspace = true } +utils = { path = "../../utils" } +hex = { version = "^0.4.3" } +serde = { workspace = true } diff --git a/rs/ic_os/network/mac_address/src/lib.rs b/rs/ic_os/network/mac_address/src/lib.rs new file mode 100644 index 00000000000..2373942d1f1 --- /dev/null +++ b/rs/ic_os/network/mac_address/src/lib.rs @@ -0,0 +1,2 @@ +pub mod mac_address; +pub mod node_type; diff --git a/rs/ic_os/network/src/mac_address.rs b/rs/ic_os/network/mac_address/src/mac_address.rs similarity index 96% rename from rs/ic_os/network/src/mac_address.rs rename to rs/ic_os/network/mac_address/src/mac_address.rs index 5c632871127..5216023a05d 100644 --- a/rs/ic_os/network/src/mac_address.rs +++ b/rs/ic_os/network/mac_address/src/mac_address.rs @@ -6,6 +6,7 @@ use regex::Regex; use sha2::{Digest, Sha256}; use crate::node_type::NodeType; +use serde::{Deserialize, Serialize}; use utils::intersperse; /// Wrapper types for MAC addresses @@ -16,9 +17,9 @@ use utils::intersperse; /// Use `.get()` to get the underlying string /// Transform between the types with `from(the_other)` // TODO - Make a canonical type which can convert to either un/formatted on demand -#[derive(Clone, Debug)] +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub struct UnformattedMacAddress(String); -#[derive(Clone, Debug)] +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)] pub struct FormattedMacAddress(String); impl UnformattedMacAddress { @@ -110,11 +111,11 @@ pub fn get_mac_address_from_ipmitool_output(output: &str) -> Result Result { // Newline added to match behavior - let seed = format!("{}{}\n", mgmt_mac.get(), deployment_name); + let seed = format!("{}{}\n", mgmt_mac.get(), deployment_environment); let vendor_part: String = hex::encode(Sha256::digest(seed)).chars().take(8).collect(); let node_index = node_type.to_char(); let mac = format!("6a0{}{}", node_index, vendor_part); diff --git a/rs/ic_os/network/src/node_type.rs b/rs/ic_os/network/mac_address/src/node_type.rs similarity index 100% rename from rs/ic_os/network/src/node_type.rs rename to rs/ic_os/network/mac_address/src/node_type.rs diff --git a/rs/ic_os/network/src/ipv6.rs b/rs/ic_os/network/src/ipv6.rs index a97e1325d50..82561e903b5 100644 --- a/rs/ic_os/network/src/ipv6.rs +++ b/rs/ic_os/network/src/ipv6.rs @@ -2,7 +2,7 @@ use std::net::Ipv6Addr; use anyhow::{anyhow, Context, Result}; -use crate::mac_address::UnformattedMacAddress; +use mac_address::mac_address::UnformattedMacAddress; use utils::intersperse; /// Generate a deterministic ipv6 address diff --git a/rs/ic_os/network/src/lib.rs b/rs/ic_os/network/src/lib.rs index 01147f7a06e..af115d4ef29 100644 --- a/rs/ic_os/network/src/lib.rs +++ b/rs/ic_os/network/src/lib.rs @@ -2,16 +2,13 @@ use std::path::Path; use anyhow::{anyhow, Result}; -use crate::mac_address::UnformattedMacAddress; use crate::systemd::generate_systemd_config_files; use config::types::{Ipv6Config, NetworkSettings}; use ipv6::generate_ipv6_address; -use mac_address::FormattedMacAddress; +use mac_address::mac_address::{FormattedMacAddress, UnformattedMacAddress}; pub mod interfaces; pub mod ipv6; -pub mod mac_address; -pub mod node_type; pub mod systemd; /// Write SetupOS or HostOS systemd network configuration. diff --git a/rs/ic_os/network/src/systemd.rs b/rs/ic_os/network/src/systemd.rs index e2d49cf247b..f5b454d17b5 100644 --- a/rs/ic_os/network/src/systemd.rs +++ b/rs/ic_os/network/src/systemd.rs @@ -6,8 +6,8 @@ use std::process::Command; use anyhow::{Context, Result}; use crate::interfaces::{get_interfaces, has_ipv6_connectivity, Interface}; -use crate::mac_address::FormattedMacAddress; use config::types::DeterministicIpv6Config; +use mac_address::mac_address::FormattedMacAddress; pub static DEFAULT_SYSTEMD_NETWORK_DIR: &str = "/run/systemd/network"; diff --git a/rs/ic_os/os_tools/hostos_tool/BUILD.bazel b/rs/ic_os/os_tools/hostos_tool/BUILD.bazel index ea5395c09e4..baeb996688b 100644 --- a/rs/ic_os/os_tools/hostos_tool/BUILD.bazel +++ b/rs/ic_os/os_tools/hostos_tool/BUILD.bazel @@ -6,6 +6,7 @@ DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", "//rs/ic_os/network", + "//rs/ic_os/network/mac_address", "//rs/ic_os/utils", "@crate_index//:anyhow", "@crate_index//:clap", diff --git a/rs/ic_os/os_tools/hostos_tool/Cargo.toml b/rs/ic_os/os_tools/hostos_tool/Cargo.toml index 08ba6eea667..88f83598de1 100644 --- a/rs/ic_os/os_tools/hostos_tool/Cargo.toml +++ b/rs/ic_os/os_tools/hostos_tool/Cargo.toml @@ -12,4 +12,5 @@ anyhow = { workspace = true } clap = { workspace = true } config = { path = "../../config" } network = { path = "../../network" } -utils = { path = "../../utils" } \ No newline at end of file +mac_address = { path = "../../network/mac_address" } +utils = { path = "../../utils" } diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 163a49abfe1..4bd0ef391da 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -5,10 +5,10 @@ use clap::{Parser, Subcommand}; use config::types::{HostOSConfig, Ipv6Config}; use config::{deserialize_config, DEFAULT_HOSTOS_CONFIG_OBJECT_PATH}; +use mac_address::mac_address::{generate_mac_address, FormattedMacAddress}; +use mac_address::node_type::NodeType; use network::generate_network_config; use network::ipv6::generate_ipv6_address; -use network::mac_address::{generate_mac_address, get_ipmi_mac, FormattedMacAddress}; -use network::node_type::NodeType; use network::systemd::DEFAULT_SYSTEMD_NETWORK_DIR; use utils::to_cidr; diff --git a/rs/ic_os/os_tools/setupos_tool/BUILD.bazel b/rs/ic_os/os_tools/setupos_tool/BUILD.bazel index 504a29a3f68..bdf6dfd71f2 100644 --- a/rs/ic_os/os_tools/setupos_tool/BUILD.bazel +++ b/rs/ic_os/os_tools/setupos_tool/BUILD.bazel @@ -6,6 +6,7 @@ DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", "//rs/ic_os/network", + "//rs/ic_os/network/mac_address", "//rs/ic_os/utils", "@crate_index//:anyhow", "@crate_index//:clap", diff --git a/rs/ic_os/os_tools/setupos_tool/Cargo.toml b/rs/ic_os/os_tools/setupos_tool/Cargo.toml index 3c62dfdb748..842affddfd3 100644 --- a/rs/ic_os/os_tools/setupos_tool/Cargo.toml +++ b/rs/ic_os/os_tools/setupos_tool/Cargo.toml @@ -12,4 +12,5 @@ anyhow = { workspace = true } clap = { workspace = true } config = { path = "../../config" } network = { path = "../../network" } -utils = { path = "../../utils" } \ No newline at end of file +mac_address = { path = "../../network/mac_address" } +utils = { path = "../../utils" } diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index f9b0640d20c..d4d8bbeb03c 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -8,10 +8,10 @@ use config::{ deserialize_config, DEFAULT_SETUPOS_CONFIG_INI_FILE_PATH, DEFAULT_SETUPOS_CONFIG_OBJECT_PATH, DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH, }; +use mac_address::mac_address::generate_mac_address; +use mac_address::node_type::NodeType; use network::generate_network_config; use network::ipv6::generate_ipv6_address; -use network::mac_address::{generate_mac_address, get_ipmi_mac, FormattedMacAddress}; -use network::node_type::NodeType; use network::systemd::DEFAULT_SYSTEMD_NETWORK_DIR; use utils::to_cidr; From e213948fc30e863c70d8e8b0a992b54fc343da69 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 22:05:15 +0000 Subject: [PATCH 091/241] Consolidate setup-hostname.sh and use config object hostname --- .../setup-hostname/hostos/setup-hostname.sh | 89 ------------------ .../setup-hostname/setup-hostname.service | 2 +- .../setup-hostname/setup-hostname.sh | 92 +++++++++++++++++-- ic-os/components/hostos.bzl | 2 +- 4 files changed, 85 insertions(+), 100 deletions(-) delete mode 100755 ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh deleted file mode 100755 index 06029ba1fc3..00000000000 --- a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -set -e - -# Set the transient or persistent hostname. - -source /opt/ic/bin/logging.sh -source /opt/ic/bin/metrics.sh - -SCRIPT="$(basename $0)[$$]" - -# Get keyword arguments -for argument in "${@}"; do - case ${argument} in - -f=* | --file=*) - FILE="${argument#*=}" - shift - ;; - -h | --help) - echo 'Usage: -Set Transient Or Persistent Hostname - -Arguments: - -f=, --file= optional: specify the file containing the node-id (Default: /boot/config/node-id) - -h, --help show this help message and exit - -t=, --type= mandatory: specify the node type (Examples: host, guest, boundary...) -' - exit 1 - ;; - -t=* | --type=*) - TYPE="${argument#*=}" - shift - ;; - *) - echo "Error: Argument is not supported." - exit 1 - ;; - esac -done - -# Set arguments if undefined -FILE="${FILE:=/boot/config/node-id}" - -function validate_arguments() { - if [ "${FILE}" == "" -o "${TYPE}" == "" ]; then - $0 --help - fi -} - -function construct_hostname() { - local mac=$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g') - - if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then - HOSTNAME=$(echo ${TYPE}-${mac}-$(cat ${FILE})) - write_log "Using hostname: ${HOSTNAME}" - write_metric "hostos_setup_hostname" \ - "1" \ - "HostOS setup hostname" \ - "gauge" - else - HOSTNAME=$(echo ${TYPE}-${mac}) - write_log "Using hostname: ${HOSTNAME}" - write_metric "hostos_setup_hostname" \ - "0" \ - "HostOS setup hostname" \ - "gauge" - fi -} - -function setup_hostname() { - if [ "$(mount | grep '/etc/hostname')" ]; then - umount /etc/hostname - fi - - if [ -d /run/ic-node/etc ]; then - echo "${HOSTNAME}" >/run/ic-node/etc/hostname - mount --bind /run/ic-node/etc/hostname /etc/hostname - restorecon -v /etc/hostname - hostname "${HOSTNAME}" - fi -} - -function main() { - validate_arguments - construct_hostname - setup_hostname -} - -main diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.service b/ic-os/components/early-boot/setup-hostname/setup-hostname.service index a1159d4104e..5f1fea3dfe6 100644 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.service +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.service @@ -11,4 +11,4 @@ WantedBy=multi-user.target [Service] Type=oneshot RemainAfterExit=true -ExecStart=/opt/ic/bin/setup-hostname.sh +ExecStart=/opt/ic/bin/setup-hostname.sh --type=guest diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh index dcbf5558fe8..d3d6253004a 100755 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh @@ -1,19 +1,93 @@ #!/bin/bash -set -ex +set -e +# Set the transient or persistent hostname. + +source /opt/ic/bin/logging.sh +source /opt/ic/bin/metrics.sh source /opt/ic/bin/config.sh +SCRIPT="$(basename $0)[$$]" + +# Get keyword arguments +for argument in "${@}"; do + case ${argument} in + -f=* | --file=*) + FILE="${argument#*=}" + shift + ;; + -h | --help) + echo 'Usage: +Set Transient Or Persistent Hostname + +Arguments: + -f=, --file= optional: specify the file containing the node-id (Default: /boot/config/node-id) + -h, --help show this help message and exit + -t=, --type= mandatory: specify the node type (Examples: host, guest, boundary...) +' + exit 1 + ;; + -t=* | --type=*) + TYPE="${argument#*=}" + shift + ;; + *) + echo "Error: Argument is not supported." + exit 1 + ;; + esac +done + +# Set arguments if undefined +FILE="${FILE:=/boot/config/node-id}" + +function validate_arguments() { + if [ "${FILE}" == "" -o "${TYPE}" == "" ]; then + $0 --help + fi +} + function read_config_variables() { - hostname=$(get_config_value '.icos_settings.hostname') - if [[ -z "${hostname}" || "${hostname}" == "null" ]]; then - hostname="unnamed" + mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') +} + +function construct_hostname() { + if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then + HOSTNAME=$(echo ${TYPE}-${mgmt_mac}-$(cat ${FILE})) + write_log "Using hostname: ${HOSTNAME}" + write_metric "setup_hostname" \ + "1" \ + "Hostname" \ + "gauge" + else + HOSTNAME=$(echo ${TYPE}-${mgmt_mac}) + write_log "Using hostname: ${HOSTNAME}" + write_metric "setup_hostname" \ + "0" \ + "Hostname" \ + "gauge" + fi +} + +function setup_hostname() { + if [ "$(mount | grep '/etc/hostname')" ]; then + umount /etc/hostname + fi + + if [ -d /run/ic-node/etc ]; then + echo "${HOSTNAME}" >/run/ic-node/etc/hostname + mount --bind /run/ic-node/etc/hostname /etc/hostname + restorecon -v /etc/hostname + hostname "${HOSTNAME}" fi } -read_config_variables +function main() { + validate_arguments + read_config_variables + construct_hostname + setup_hostname +} -echo "${hostname}" >/run/ic-node/etc/hostname -mount --bind /run/ic-node/etc/hostname /etc/hostname -restorecon -v /etc/hostname -hostname "${hostname}" +main diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 339323557e9..efbd147cfb8 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -35,7 +35,7 @@ component_files = { # early-boot Label("early-boot/relabel-machine-id/relabel-machine-id.sh"): "/opt/ic/bin/relabel-machine-id.sh", Label("early-boot/relabel-machine-id/relabel-machine-id.service"): "/etc/systemd/system/relabel-machine-id.service", - Label("early-boot/setup-hostname/hostos/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", + Label("early-boot/setup-hostname/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", Label("early-boot/setup-hostname/hostos/setup-hostname.service"): "/etc/systemd/system/setup-hostname.service", Label("early-boot/setup-hostname/hostname-empty"): "/etc/hostname", Label("early-boot/save-machine-id/save-machine-id.sh"): "/opt/ic/bin/save-machine-id.sh", From f65556e0bd04fe94523998974213c89ab143ef5c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 22:06:20 +0000 Subject: [PATCH 092/241] Delete unused FetchMacAddress command --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 4bd0ef391da..9a709cdf9d3 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -28,7 +28,6 @@ pub enum Commands { #[arg(short, long, default_value = "HostOS")] node_type: String, }, - FetchMacAddress {}, } #[derive(Parser)] @@ -123,13 +122,6 @@ pub fn main() -> Result<()> { println!("{}", generated_mac); Ok(()) } - Some(Commands::FetchMacAddress {}) => { - let hostos_config: HostOSConfig = - deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; - - println!("{}", hostos_config.icos_settings.mgmt_mac); - Ok(()) - } None => Err(anyhow!( "No subcommand specified. Run with '--help' for subcommands" )), From 348716247f3b96a8c22d1356f23e6b3787722add Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 22:08:20 +0000 Subject: [PATCH 093/241] Fix reference to mac_address crate --- rs/ic_os/config/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ic_os/config/BUILD.bazel b/rs/ic_os/config/BUILD.bazel index 6e2ca170df8..5e4ed19f5dd 100644 --- a/rs/ic_os/config/BUILD.bazel +++ b/rs/ic_os/config/BUILD.bazel @@ -4,7 +4,7 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. - "//rs/ic_os/network/config", + "//rs/ic_os/network/mac_address", "//rs/ic_os/utils", "//rs/types/types", "@crate_index//:anyhow", From 77817d335242f5fb72bc305463a1ae70bde8a2d3 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 17 Oct 2024 22:35:41 +0000 Subject: [PATCH 094/241] Remove colons in mgmt_mac --- ic-os/components/early-boot/setup-hostname/setup-hostname.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh index d3d6253004a..0f4c7ab10cd 100755 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh @@ -50,6 +50,7 @@ function validate_arguments() { function read_config_variables() { mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') + mgmt_mac=${mgmt_mac//:/} # Remove colons from mgmt_mac } function construct_hostname() { From 6779607f00ac3b10878b3ebe8be78d62a4529529 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 15:26:48 +0000 Subject: [PATCH 095/241] Fix buildifier --- rs/ic_os/network/BUILD.bazel | 2 +- rs/ic_os/network/mac_address/BUILD.bazel | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/ic_os/network/BUILD.bazel b/rs/ic_os/network/BUILD.bazel index c31a236e9ae..9ce753e420f 100644 --- a/rs/ic_os/network/BUILD.bazel +++ b/rs/ic_os/network/BUILD.bazel @@ -5,8 +5,8 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", - "//rs/ic_os/utils", "//rs/ic_os/network/mac_address", + "//rs/ic_os/utils", "@crate_index//:anyhow", "@crate_index//:ping", "@crate_index//:rayon", diff --git a/rs/ic_os/network/mac_address/BUILD.bazel b/rs/ic_os/network/mac_address/BUILD.bazel index 31de5a1bf9e..5ac0a6afb39 100644 --- a/rs/ic_os/network/mac_address/BUILD.bazel +++ b/rs/ic_os/network/mac_address/BUILD.bazel @@ -6,10 +6,10 @@ DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/utils", "@crate_index//:anyhow", - "@crate_index//:regex", - "@crate_index//:sha2", "@crate_index//:hex", + "@crate_index//:regex", "@crate_index//:serde", + "@crate_index//:sha2", ] rust_library( From ddb37cfcb07610850351248c30763043cf89b5b2 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 15:31:45 +0000 Subject: [PATCH 096/241] Add type description --- rs/ic_os/config/src/types.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 83a2258b1d0..7fc45ef39ad 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -84,7 +84,10 @@ pub struct BackupSpoolSettings { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct ICOSSettings { + /// in nested testing, mgmt_mac is set in deployment.json.template, + /// else found dynamically in call to config tool CreateSetuposConfig pub mgmt_mac: FormattedMacAddress, + /// "mainnet" or "testnet" pub deployment_environment: String, pub logging: Logging, /// This file must be a text file containing the public key of the NNS to be used. From 248abc5669be78ad74d163f81563968296b312dd Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 20:02:54 +0000 Subject: [PATCH 097/241] Remove unnecessary network.conf references --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 2 +- .../init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 8264328e962..24cd27a5e6e 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -111,7 +111,7 @@ function build_ic_bootstrap_tar() { local BOOTSTRAP_TMPDIR=$(mktemp -d) - # todo: delete network.conf and malicious_behaviour.conf + # todo: delete malicious_behaviour.conf if [ "${MALICIOUS_BEHAVIOR}" != "" ]; then echo "malicious_behavior=${MALICIOUS_BEHAVIOR}" >"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" fi diff --git a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh index 0ca1f22480a..2f717785a30 100755 --- a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh @@ -102,7 +102,7 @@ function process_bootstrap() { # stash the following configuration files to config store # note: keep this list in sync with configurations supported in build-bootstrap-config-image.sh - for FILE in network.conf malicious_behavior.conf config.json; do + for FILE in malicious_behavior.conf config.json; do if [ -e "${TMPDIR}/${FILE}" ]; then echo "Setting up ${FILE}" cp "${TMPDIR}/${FILE}" "${CONFIG_ROOT}/${FILE}" From 1014760b599791e8807fcd6389ebc876f1be2a32 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 21:16:43 +0000 Subject: [PATCH 098/241] Update guestos_tool to use the config object --- rs/ic_os/config/src/config_ini.rs | 2 +- rs/ic_os/config/src/lib.rs | 10 +- .../src/generate_network_config.rs | 111 ++++-------------- rs/ic_os/os_tools/guestos_tool/src/main.rs | 38 +++--- 4 files changed, 53 insertions(+), 108 deletions(-) diff --git a/rs/ic_os/config/src/config_ini.rs b/rs/ic_os/config/src/config_ini.rs index 4bc6c92219c..518da5d5058 100644 --- a/rs/ic_os/config/src/config_ini.rs +++ b/rs/ic_os/config/src/config_ini.rs @@ -116,7 +116,7 @@ fn parse_config_line(line: &str) -> Option<(String, String)> { } } -pub fn config_map_from_path(config_file_path: &Path) -> Result { +fn config_map_from_path(config_file_path: &Path) -> Result { let file_contents = read_to_string(config_file_path) .with_context(|| format!("Error reading file: {}", config_file_path.display()))?; diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 347ee0f9ab6..27007012027 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -20,6 +20,7 @@ pub static DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/con pub static DEFAULT_HOSTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config.json"; pub static DEFAULT_HOSTOS_GUESTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config-guestos.json"; +pub static DEFAULT_GUESTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config.json"; pub fn serialize_and_write_config(path: &Path, config: &T) -> Result<()> { let serialized_config = @@ -34,11 +35,12 @@ pub fn serialize_and_write_config(path: &Path, config: &T) -> Resu Ok(()) } -pub fn deserialize_config Deserialize<'de>>(file_path: &str) -> Result { - let file = File::open(file_path).context(format!("Failed to open file: {}", file_path))?; +pub fn deserialize_config Deserialize<'de>, P: AsRef>(file_path: P) -> Result { + let file = + File::open(&file_path).context(format!("Failed to open file: {:?}", file_path.as_ref()))?; serde_json::from_reader(file).context(format!( - "Failed to deserialize JSON from file: {}", - file_path + "Failed to deserialize JSON from file: {:?}", + file_path.as_ref() )) } diff --git a/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs b/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs index 0f4eb2b2cb3..7440d80f595 100644 --- a/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs +++ b/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::fs::write; use std::net::{Ipv4Addr, Ipv6Addr}; use std::path::{Path, PathBuf}; @@ -6,14 +5,12 @@ use std::str::FromStr; use anyhow::{bail, Context, Result}; -use config::config_ini::config_map_from_path; +use config::types::Ipv6Config; use network::interfaces::{get_interface_name as get_valid_interface_name, get_interface_paths}; use utils::get_command_stdout; use network::systemd::IPV6_NAME_SERVER_NETWORKD_CONTENTS; -pub static DEFAULT_GUESTOS_NETWORK_CONFIG_PATH: &str = "/boot/config/network.conf"; - const IPV4_NAME_SERVER_NETWORKD_CONTENTS: &str = "DNS=1.1.1.1\nDNS=1.0.0.1\nDNS=8.8.8.8\nDNS=8.8.4.4\n"; @@ -85,23 +82,20 @@ impl IpAddressInfo { /// Generate network configuration for systemd networkd based on the provided network configuration. pub fn generate_networkd_config( - network_config: &Path, + ipv6_config: Ipv6Config, systemd_network_dir: &Path, ipv4_info: Option, ) -> Result<()> { - eprintln!("Network config file: {}", network_config.display()); + eprintln!("IPv6 config info: {:?}", ipv6_config); + eprintln!("IPv4 address info: {:?}", ipv4_info); eprintln!( "Systemd network directory: {}", systemd_network_dir.display() ); - eprintln!("IPv4 address info: {:?}", ipv4_info); std::fs::create_dir_all(systemd_network_dir)?; - let network_config_variables: HashMap = config_map_from_path(network_config)?; - eprintln!("Network parameters {:#?}", network_config_variables); - - let network_info: NetworkInfo = create_network_info(&network_config_variables, ipv4_info)?; + let network_info: NetworkInfo = create_network_info(ipv6_config, ipv4_info)?; eprintln!("{:#?}", network_info); let network_interface_name = get_interface_name()?; @@ -148,21 +142,18 @@ pub fn validate_and_construct_ipv4_address_info( } fn create_network_info( - network_config_variables: &HashMap, + ipv6_config: Ipv6Config, ipv4_info: Option, ) -> Result { - let ipv6_info = match ( - network_config_variables.get("ipv6_address"), - network_config_variables.get("ipv6_gateway"), - ) { - (Some(ipv6_address_with_prefix), Some(ipv6_gateway)) => { - process_ipv6_address_and_gateway(ipv6_address_with_prefix, ipv6_gateway)? + let ipv6_info = match ipv6_config { + Ipv6Config::RouterAdvertisement => None, + Ipv6Config::Deterministic(_) => { + bail!("GuestOSConfig Ipv6Config should not be of type Deterministic."); } - (Some(_), None) | (None, Some(_)) => { - // Either IPv6 address or gateway is provided, but not both - bail!("ERROR: Incomplete configuration - both an IPv6 address and a gateway are required. Please specify both."); - } - _ => None, + Ipv6Config::Fixed(ipv6_config) => Some(IpAddressInfo::new_ipv6_address( + &ipv6_config.address, + &ipv6_config.gateway.to_string(), + )?), }; Ok(NetworkInfo { @@ -171,21 +162,6 @@ fn create_network_info( }) } -fn process_ipv6_address_and_gateway( - ipv6_address_with_prefix: &str, - ipv6_gateway: &str, -) -> Result> { - if ipv6_address_with_prefix.is_empty() && ipv6_gateway.is_empty() { - eprintln!("Both IPv6 address and gateway are unspecified. Proceeding with network configuration using Router Advertisements."); - Ok(None) - } else { - Ok(Some(IpAddressInfo::new_ipv6_address( - ipv6_address_with_prefix, - ipv6_gateway, - )?)) - } -} - fn generate_networkd_config_contents( network_info: NetworkInfo, interface_name: &str, @@ -299,19 +275,18 @@ fn is_k8s_testnet() -> Result { #[cfg(test)] mod tests { use super::*; + use config::types::*; #[test] fn test_create_network_info_with_valid_ipv6_and_ipv4() { - let mut network_config_variables = HashMap::new(); - network_config_variables.insert("ipv6_address".to_string(), "2001:db8::1/64".to_string()); - network_config_variables.insert("ipv6_gateway".to_string(), "2001:db8::1".to_string()); - - eprintln!("network_config_variables: {:?}", network_config_variables); - let ipv4_info = Some(IpAddressInfo::new_ipv4_address("192.168.1.100", "30", "192.168.1.1").unwrap()); + let ipv6_config = Ipv6Config::Fixed(FixedIpv6Config { + address: "2001:db8::1/64".to_string(), + gateway: "2001:db8::1".parse().unwrap(), + }); - let result = create_network_info(&network_config_variables, ipv4_info).unwrap(); + let result = create_network_info(ipv6_config, ipv4_info).unwrap(); assert!(result.ipv6_info.is_some()); let ipv6_info = result.ipv6_info.as_ref().unwrap(); @@ -325,15 +300,13 @@ mod tests { #[test] fn test_create_network_info_with_valid_ipv6_and_no_ipv4() { - let mut network_config_variables = HashMap::new(); - network_config_variables.insert("ipv6_address".to_string(), "2001:db8::1/64".to_string()); - network_config_variables.insert("ipv6_gateway".to_string(), "2001:db8::1".to_string()); - - eprintln!("network_config_variables: {:?}", network_config_variables); - + let ipv6_config = Ipv6Config::Fixed(FixedIpv6Config { + address: "2001:db8::1/64".to_string(), + gateway: "2001:db8::1".parse().unwrap(), + }); let ipv4_info = None; - let result = create_network_info(&network_config_variables, ipv4_info).unwrap(); + let result = create_network_info(ipv6_config, ipv4_info).unwrap(); assert!(result.ipv6_info.is_some()); let ipv6_info = result.ipv6_info.as_ref().unwrap(); @@ -341,40 +314,6 @@ mod tests { assert_eq!(ipv6_info.gateway, "2001:db8::1"); } - #[test] - fn test_create_network_info_with_invalid_ipv6() { - let mut network_config_variables = HashMap::new(); - network_config_variables.insert("ipv6_address".to_string(), "invalid_address".to_string()); - network_config_variables.insert("ipv6_gateway".to_string(), "invalid_gateway".to_string()); - - let result = create_network_info(&network_config_variables, None); - - assert!(result.is_err(), "Invalid ipv6 address configuration"); - } - - #[test] - fn test_create_network_info_with_missing_ipv6_gateway() { - let mut network_config_variables = HashMap::new(); - network_config_variables.insert("ipv6_address".to_string(), "invalid_address".to_string()); - // ipv6 gateway intentionally omitted: - // network_config_variables.insert("ipv6_gateway".to_string(), "invalid_gateway".to_string()); - - let result = create_network_info(&network_config_variables, None); - - assert!( - result.is_err(), - "Expected an error when IPv6 gateway is missing" - ); - } - - #[test] - fn test_create_network_info_without_ipv6_or_ipv4_or_nameservers() { - let network_config_variables = HashMap::new(); - - let result = create_network_info(&network_config_variables, None).unwrap(); - assert!(result.ipv6_info.is_none()); - } - #[test] fn test_validate_ipv4_network_info_no_input() { assert!(validate_and_construct_ipv4_address_info(None, None, None) diff --git a/rs/ic_os/os_tools/guestos_tool/src/main.rs b/rs/ic_os/os_tools/guestos_tool/src/main.rs index d3206585e17..977843f4e0f 100644 --- a/rs/ic_os/os_tools/guestos_tool/src/main.rs +++ b/rs/ic_os/os_tools/guestos_tool/src/main.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use anyhow::Result; use clap::{Parser, Subcommand}; @@ -10,11 +10,10 @@ mod prometheus_metric; use prometheus_metric::write_single_metric; mod generate_network_config; -use generate_network_config::{ - generate_networkd_config, validate_and_construct_ipv4_address_info, - DEFAULT_GUESTOS_NETWORK_CONFIG_PATH, -}; +use generate_network_config::{generate_networkd_config, validate_and_construct_ipv4_address_info}; +use config::deserialize_config; +use config::types::GuestOSConfig; use network::systemd::{restart_systemd_networkd, DEFAULT_SYSTEMD_NETWORK_DIR}; #[derive(Subcommand)] @@ -25,9 +24,9 @@ pub enum Commands { /// systemd-networkd output directory systemd_network_dir: String, - #[arg(long, default_value_t = DEFAULT_GUESTOS_NETWORK_CONFIG_PATH.to_string(), value_name = "FILE")] + #[arg(long, default_value = config::DEFAULT_GUESTOS_CONFIG_OBJECT_PATH, value_name = "FILE")] /// network.conf input file - network_config: String, + config_object: PathBuf, }, /// Regenerate systemd network configuration files, optionally incorporating specified IPv4 configuration parameters, and then restart the systemd network. RegenerateNetworkConfig { @@ -35,9 +34,9 @@ pub enum Commands { /// systemd-networkd output directory systemd_network_dir: String, - #[arg(long, default_value_t = DEFAULT_GUESTOS_NETWORK_CONFIG_PATH.to_string(), value_name = "FILE")] + #[arg(long, default_value = config::DEFAULT_GUESTOS_CONFIG_OBJECT_PATH, value_name = "FILE")] /// network.conf input file - network_config: String, + config_object: PathBuf, #[arg(long, value_name = "IPV4_ADDRESS")] /// IPv4 address @@ -84,15 +83,19 @@ pub fn main() -> Result<()> { } Some(Commands::GenerateNetworkConfig { systemd_network_dir, - network_config, - }) => generate_networkd_config( - Path::new(&network_config), - Path::new(&systemd_network_dir), - None, - ), + config_object, + }) => { + let guestos_config: GuestOSConfig = deserialize_config(config_object)?; + generate_networkd_config( + guestos_config.network_settings.ipv6_config, + Path::new(&systemd_network_dir), + None, + ) + } + Some(Commands::RegenerateNetworkConfig { systemd_network_dir, - network_config, + config_object, ipv4_address, ipv4_prefix_length, ipv4_gateway, @@ -103,8 +106,9 @@ pub fn main() -> Result<()> { ipv4_gateway.as_deref(), )?; + let guestos_config: GuestOSConfig = deserialize_config(config_object)?; generate_networkd_config( - Path::new(&network_config), + guestos_config.network_settings.ipv6_config, Path::new(&systemd_network_dir), ipv4_info, )?; From 3da367a18a119b6bdaf2d18facf71524d6788c13 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 21:24:30 +0000 Subject: [PATCH 099/241] Remove old, unnecessary todo --- ic-os/components/ic/generate-replica-config.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index feea11b0c93..19bb1eeff45 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -110,8 +110,6 @@ function set_default_config_values() { [ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default value is 24h [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default value is 1h [ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # Default is 600 blocks (around 10min) - - # TODO: If the Jaeger address is not specified the config file will contain Some(""). This needs to be fixed. [ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" # todo: remove node_index variable and hard-code into ic.json5.template From 468b9a969186be63551ac44f8211e1bae26f0520 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 21:30:37 +0000 Subject: [PATCH 100/241] Update bootstrap-ic-node comment --- .../init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh index 2f717785a30..1c2bebacccb 100755 --- a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh @@ -69,8 +69,8 @@ function find_config_devices() { } # Process the bootstrap package given as first argument to populate -# both config space and -# parts of /var/lib/ic/data and /var/lib/ic/crypto +# both config space and parts of /var/lib/ic/data and /var/lib/ic/crypto +# note: keep this list in sync with configurations supported in build-bootstrap-config-image.sh # # Arguments: # - $1: path to the bootstrap package (typically /mnt/ic-bootstrap.tar) @@ -100,8 +100,6 @@ function process_bootstrap() { fi done - # stash the following configuration files to config store - # note: keep this list in sync with configurations supported in build-bootstrap-config-image.sh for FILE in malicious_behavior.conf config.json; do if [ -e "${TMPDIR}/${FILE}" ]; then echo "Setting up ${FILE}" From d11c584f6b78406c0b0f1f7e103297ad1280a5cb Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 21:32:42 +0000 Subject: [PATCH 101/241] Update GuestOS config partition documentation --- ic-os/docs/Configuration.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index 8d42ce11442..5d41f53a135 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -44,14 +44,14 @@ The reason for the bootstrap config image redirection is to ensure that GuestOS == GuestOS config partition -TODO: update... - The config partition stores information that must be preserved across system upgrades and needs to be available during early boot time. Consequently, this information cannot reside within the encrypted payload data partition. Currently, all contents in the config partition are stored as plain-text without integrity protection. These files are stored in `/boot/config` or `/var/lib/ic`. To see where each configuration file is stored, refer to link:../../components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh[bootstrap-ic-node] +Other config partition files produced by the GuestOS at runtime: + === CONFIGURED file This file serves as a tag to indicate that the one-time bootstrap configuration has been completed. If the `/boot/config/CONFIGURED` file is not present, the boot sequence will search for a virtual USB stick (the bootstrap config image) containing the injected configuration files, and create the file. From 0f7c637aec86f5836bf1f82f7ed863d91850ebb3 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 18 Oct 2024 21:52:21 +0000 Subject: [PATCH 102/241] Use malicious_behavior in config_tool --- .../build-bootstrap-config-image.sh | 16 ---------- .../components/ic/generate-replica-config.sh | 31 ++----------------- ic-os/components/ic/ic-replica.service | 2 +- .../guestos/bootstrap-ic-node.sh | 2 +- 4 files changed, 4 insertions(+), 47 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 24cd27a5e6e..4350eedbfd2 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -46,13 +46,6 @@ options may be specified: --node_operator_private_key path Should point to a file containing a Node Provider private key PEM. - - --malicious_behavior malicious_behavior - A JSON-object that describes the malicious behavior activated on - the node. This is only used for testing. - - The Json-object corresponds to this Rust-structure: - ic_types::malicious_behaviour::MaliciousBehaviour EOF } @@ -67,7 +60,6 @@ function build_ic_bootstrap_tar() { local IC_CRYPTO IC_STATE IC_REGISTRY_LOCAL_STORE local NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY local ACCOUNTS_SSH_AUTHORIZED_KEYS - local MALICIOUS_BEHAVIOR while true; do if [ $# == 0 ]; then @@ -96,9 +88,6 @@ function build_ic_bootstrap_tar() { --node_operator_private_key) NODE_OPERATOR_PRIVATE_KEY="$2" ;; - --malicious_behavior) - MALICIOUS_BEHAVIOR="$2" - ;; *) echo "Unrecognized option: $1" usage @@ -111,11 +100,6 @@ function build_ic_bootstrap_tar() { local BOOTSTRAP_TMPDIR=$(mktemp -d) - # todo: delete malicious_behaviour.conf - if [ "${MALICIOUS_BEHAVIOR}" != "" ]; then - echo "malicious_behavior=${MALICIOUS_BEHAVIOR}" >"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" - fi - # todo: switch nns_public_key.pem, node_operator_private_key.pem. and accounts_ssh to use config object if [ "${NNS_PUBLIC_KEY}" != "" ]; then cp "${NNS_PUBLIC_KEY}" "${BOOTSTRAP_TMPDIR}/nns_public_key.pem" diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 19bb1eeff45..887320581ee 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -12,8 +12,6 @@ Usage: Generate replica config from template file. - -m malicious_behavior.conf: Optional, malicious behavior parameters - -i infile: input ic.json5.template file -o outfile: output ic.json5 file EOF @@ -25,9 +23,7 @@ function read_config_variables() { BACKUP_PURGING_INTERVAL_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_purging_interval_seconds') QUERY_STATS_EPOCH_LENGTH=$(get_config_value '.guestos_settings.guestos_dev_settings.query_stats_epoch_length') JAEGER_ADDR=$(get_config_value '.guestos_settings.guestos_dev_settings.jaeger_addr') - - # todo: - # "malicious_behavior") malicious_behavior="${value}" ;; + MALICIOUS_BEHAVIOR=$(get_config_value '.guestos_settings.guestos_dev_settings.malicious_behavior') } function configure_ipv6() { @@ -116,27 +112,8 @@ function set_default_config_values() { NODE_INDEX="0" } -# Read malicious behavior config variables from file. The file must be of the -# form "key=value" for each line with a specific set of keys permissible (see -# code below). -# -# Arguments: -# - $1: Name of the file to be read. -function read_malicious_behavior_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "malicious_behavior") malicious_behavior="${value}" ;; - esac - done <"$1" -} - -while getopts "m:i:o:" OPT; do +while getopts "i:o:" OPT; do case "${OPT}" in - m) - MALICIOUS_BEHAVIOR_CONFIG_FILE="${OPTARG}" - ;; i) IN_FILE="${OPTARG}" ;; @@ -158,10 +135,6 @@ fi configure_ipv6 configure_ipv4 -if [ "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" != "" -a -e "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" ]; then - read_malicious_behavior_variables "${MALICIOUS_BEHAVIOR_CONFIG_FILE}" -fi - read_config_variables set_default_config_values diff --git a/ic-os/components/ic/ic-replica.service b/ic-os/components/ic/ic-replica.service index 293a0b88625..9ce156ffdc8 100644 --- a/ic-os/components/ic/ic-replica.service +++ b/ic-os/components/ic/ic-replica.service @@ -28,7 +28,7 @@ Environment=RUST_BACKTRACE=1 # Remember to update 'rs/default.nix' for nix-shell users # Remember to update 'src/dfx/src/actors/replica.rs' in the sdk repo for dfx users Environment=RUST_MIN_STACK=8192000 -ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -m /boot/config/malicious_behavior.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 +ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 ExecStart=/opt/ic/bin/orchestrator --replica-binary-dir /var/lib/ic/data/images --cup-dir /var/lib/ic/data/cups --replica-config-file /run/ic-node/config/ic.json5 --enable-provisional-registration --ic-binary-directory /opt/ic/bin --orchestrator-data-directory /var/lib/ic/data/orchestrator --version-file /opt/ic/share/version.txt LimitNOFILE=16777216 Restart=always diff --git a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh index 1c2bebacccb..7be1d64f502 100755 --- a/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/guestos/bootstrap-ic-node.sh @@ -100,7 +100,7 @@ function process_bootstrap() { fi done - for FILE in malicious_behavior.conf config.json; do + for FILE in config.json; do if [ -e "${TMPDIR}/${FILE}" ]; then echo "Setting up ${FILE}" cp "${TMPDIR}/${FILE}" "${CONFIG_ROOT}/${FILE}" From 4ee764e02f708f0ee7f490866826923eb207866b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 21 Oct 2024 22:25:06 +0000 Subject: [PATCH 103/241] Create GenerateTestnetConfig config command --- .../config/src/generate_testnet_config.rs | 253 ++++++++++++++++++ rs/ic_os/config/src/lib.rs | 1 + rs/ic_os/config/src/main.rs | 117 +++++++- 3 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 rs/ic_os/config/src/generate_testnet_config.rs diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs new file mode 100644 index 00000000000..ab3af2d6808 --- /dev/null +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -0,0 +1,253 @@ +use anyhow::Result; +use mac_address::mac_address::FormattedMacAddress; +use std::net::{Ipv4Addr, Ipv6Addr}; +use std::path::PathBuf; +use url::Url; + +use crate::serialize_and_write_config; +use crate::types::*; + +pub struct GenerateTestnetConfigArgs { + // NetworkSettings arguments + pub ipv6_config_type: Option, // "Deterministic", "Fixed", "RouterAdvertisement" + pub deterministic_prefix: Option, + pub deterministic_prefix_length: Option, + pub deterministic_gateway: Option, + pub fixed_address: Option, + pub fixed_gateway: Option, + pub ipv4_address: Option, + pub ipv4_gateway: Option, + pub ipv4_prefix_length: Option, + pub ipv4_domain: Option, + + // ICOSSettings arguments + pub mgmt_mac: Option, + pub deployment_environment: Option, + pub elasticsearch_hosts: Option, + pub elasticsearch_tags: Option, + pub nns_public_key_path: Option, + pub nns_urls: Option>, + pub node_operator_private_key_path: Option, + pub ssh_authorized_keys_path: Option, + + // GuestOSSettings arguments + pub ic_crypto_path: Option, + pub ic_state_path: Option, + pub ic_registry_local_store_path: Option, + + // GuestOSDevSettings arguments + pub backup_retention_time_seconds: Option, + pub backup_purging_interval_seconds: Option, + pub malicious_behavior: Option, + pub query_stats_epoch_length: Option, + pub bitcoind_addr: Option, + pub jaeger_addr: Option, + pub socks_proxy: Option, + + // Output path + pub guestos_config_json_path: PathBuf, +} + +/// Generates a writes a serialized GuestOSConfig to guestos_config_json_path +/// Any required config fields that aren't specified will receive dummy values +pub fn generate_testnet_config(args: GenerateTestnetConfigArgs) -> Result<()> { + let GenerateTestnetConfigArgs { + ipv6_config_type, + deterministic_prefix, + deterministic_prefix_length, + deterministic_gateway, + fixed_address, + fixed_gateway, + ipv4_address, + ipv4_gateway, + ipv4_prefix_length, + ipv4_domain, + mgmt_mac, + deployment_environment, + elasticsearch_hosts, + elasticsearch_tags, + nns_public_key_path, + nns_urls, + node_operator_private_key_path, + ssh_authorized_keys_path, + ic_crypto_path, + ic_state_path, + ic_registry_local_store_path, + backup_retention_time_seconds, + backup_purging_interval_seconds, + malicious_behavior, + query_stats_epoch_length, + bitcoind_addr, + jaeger_addr, + socks_proxy, + guestos_config_json_path, + } = args; + + // Construct the NetworkSettings + let ipv6_config = match ipv6_config_type.as_deref() { + Some("Deterministic") => { + let prefix = deterministic_prefix.ok_or_else(|| { + anyhow::anyhow!( + "deterministic_prefix is required when ipv6_config_type is 'Deterministic'" + ) + })?; + let prefix_length = deterministic_prefix_length.ok_or_else(|| { + anyhow::anyhow!( + "deterministic_prefix_length is required when ipv6_config_type is 'Deterministic'" + ) + })?; + let gateway_str = deterministic_gateway.ok_or_else(|| { + anyhow::anyhow!( + "deterministic_gateway is required when ipv6_config_type is 'Deterministic'" + ) + })?; + let gateway = gateway_str + .parse::() + .map_err(|e| anyhow::anyhow!("Failed to parse deterministic_gateway: {}", e))?; + + Ipv6Config::Deterministic(DeterministicIpv6Config { + prefix, + prefix_length, + gateway, + }) + } + Some("Fixed") => { + let address = fixed_address.ok_or_else(|| { + anyhow::anyhow!("fixed_address is required when ipv6_config_type is 'Fixed'") + })?; + let gateway_str = fixed_gateway.ok_or_else(|| { + anyhow::anyhow!("fixed_gateway is required when ipv6_config_type is 'Fixed'") + })?; + let gateway = gateway_str + .parse::() + .map_err(|e| anyhow::anyhow!("Failed to parse fixed_gateway: {}", e))?; + + Ipv6Config::Fixed(FixedIpv6Config { address, gateway }) + } + // Default to RouterAdvertisement if not provided + Some("RouterAdvertisement") | None => Ipv6Config::RouterAdvertisement, + Some(other) => { + anyhow::bail!("Invalid ipv6_config_type '{}'. Must be 'Deterministic', 'Fixed', or 'RouterAdvertisement'.", other); + } + }; + + let ipv4_config = match (ipv4_address, ipv4_gateway, ipv4_prefix_length, ipv4_domain) { + (Some(addr_str), Some(gw_str), Some(prefix_len), Some(domain)) => Some(Ipv4Config { + address: addr_str + .parse::() + .map_err(|e| anyhow::anyhow!("Failed to parse ipv4_address: {}", e))?, + gateway: gw_str + .parse::() + .map_err(|e| anyhow::anyhow!("Failed to parse ipv4_gateway: {}", e))?, + prefix_length: prefix_len, + domain, + }), + (None, None, None, None) => None, + _ => { + anyhow::bail!("Incomplete IPv4 configuration provided. All parameters (ipv4_address, ipv4_gateway, ipv4_prefix_length, ipv4_domain) are required for IPv4 configuration."); + } + }; + + let network_settings = NetworkSettings { + ipv6_config, + ipv4_config, + }; + + // Construct ICOSSettings + let mgmt_mac = match mgmt_mac { + Some(mac_str) => FormattedMacAddress::try_from(mac_str.as_str())?, + None => { + // Use a dummy MAC address + FormattedMacAddress::try_from("00:00:00:00:00:00")? + } + }; + + let deployment_environment = deployment_environment.unwrap_or_else(|| "testnet".to_string()); + + let logging = Logging { + elasticsearch_hosts: elasticsearch_hosts.unwrap_or_else(|| "".to_string()), + elasticsearch_tags, + }; + + let nns_public_key_path = + nns_public_key_path.unwrap_or_else(|| PathBuf::from("/path/to/nns_public_key.pem")); + + let nns_urls = match nns_urls { + Some(urls) => { + let parsed_urls = urls + .iter() + .map(|s| Url::parse(s)) + .collect::, _>>()?; + if parsed_urls.is_empty() { + vec![Url::parse("http://localhost")?] + } else { + parsed_urls + } + } + None => vec![Url::parse("http://localhost")?], + }; + + let icos_settings = ICOSSettings { + mgmt_mac, + deployment_environment, + logging, + nns_public_key_path, + nns_urls, + node_operator_private_key_path, + ssh_authorized_keys_path, + icos_dev_settings: ICOSDevSettings::default(), + }; + + // Construct GuestOSDevSettings + let backup_spool = + if backup_retention_time_seconds.is_some() || backup_purging_interval_seconds.is_some() { + Some(BackupSpoolSettings { + backup_retention_time_seconds, + backup_purging_interval_seconds, + }) + } else { + None + }; + + let malicious_behavior = if let Some(mb_str) = malicious_behavior { + Some(serde_json::from_str(&mb_str)?) + } else { + None + }; + + let guestos_dev_settings = GuestOSDevSettings { + backup_spool, + malicious_behavior, + query_stats_epoch_length, + bitcoind_addr, + jaeger_addr, + socks_proxy, + }; + + // Construct GuestOSSettings + let guestos_settings = GuestOSSettings { + ic_crypto_path, + ic_state_path, + ic_registry_local_store_path, + guestos_dev_settings, + }; + + // Assemble GuestOSConfig + let guestos_config = GuestOSConfig { + network_settings, + icos_settings, + guestos_settings, + }; + + println!("GuestOSConfig: {:?}", guestos_config); + + // Write the configuration to a file + serialize_and_write_config(&guestos_config_json_path, &guestos_config)?; + + println!( + "GuestOSConfig has been written to {}", + guestos_config_json_path.display() + ); + + Ok(()) +} diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 27007012027..5b3f3c8e67d 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -1,5 +1,6 @@ pub mod config_ini; pub mod deployment_json; +pub mod generate_testnet_config; pub mod types; use anyhow::{Context, Result}; diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index ae05330f15b..011a12d37b0 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use clap::{Parser, Subcommand}; +use clap::{Args, Parser, Subcommand}; use config::config_ini::{get_config_ini_settings, ConfigIniSettings}; use config::deployment_json::get_deployment_settings; use config::serialize_and_write_config; @@ -7,9 +7,11 @@ use mac_address::mac_address::{get_ipmi_mac, FormattedMacAddress}; use std::fs::File; use std::path::{Path, PathBuf}; +use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; use config::types::*; #[derive(Subcommand)] +#[allow(clippy::large_enum_variant)] pub enum Commands { /// Creates SetupOSConfig object CreateSetuposConfig { @@ -47,6 +49,8 @@ pub enum Commands { #[arg(long, value_name = "ipv6_address")] guestos_ipv6_address: String, }, + /// Creates a GuestOSConfig object directly from GenerateTestnetConfigClapArgs + GenerateTestnetConfig(GenerateTestnetConfigClapArgs), } #[derive(Parser)] @@ -56,6 +60,76 @@ struct ConfigArgs { command: Option, } +#[derive(Args)] +pub struct GenerateTestnetConfigClapArgs { + #[arg(long)] + pub ipv6_config_type: Option, // "Deterministic", "Fixed", "RouterAdvertisement" + #[arg(long)] + pub deterministic_prefix: Option, + #[arg(long)] + pub deterministic_prefix_length: Option, + #[arg(long)] + pub deterministic_gateway: Option, + #[arg(long)] + pub fixed_address: Option, + #[arg(long)] + pub fixed_gateway: Option, + #[arg(long)] + pub ipv4_address: Option, + #[arg(long)] + pub ipv4_gateway: Option, + #[arg(long)] + pub ipv4_prefix_length: Option, + #[arg(long)] + pub ipv4_domain: Option, + + // ICOSSettings arguments + #[arg(long)] + pub mgmt_mac: Option, + #[arg(long)] + pub deployment_environment: Option, + #[arg(long)] + pub elasticsearch_hosts: Option, + #[arg(long)] + pub elasticsearch_tags: Option, + #[arg(long)] + pub nns_public_key_path: Option, + #[arg(long)] + pub nns_urls: Option>, + #[arg(long)] + pub node_operator_private_key_path: Option, + #[arg(long)] + pub ssh_authorized_keys_path: Option, + + // GuestOSSettings arguments + #[arg(long)] + pub ic_crypto_path: Option, + #[arg(long)] + pub ic_state_path: Option, + #[arg(long)] + pub ic_registry_local_store_path: Option, + + // GuestOSDevSettings arguments + #[arg(long)] + pub backup_retention_time_seconds: Option, + #[arg(long)] + pub backup_purging_interval_seconds: Option, + #[arg(long)] + pub malicious_behavior: Option, + #[arg(long)] + pub query_stats_epoch_length: Option, + #[arg(long)] + pub bitcoind_addr: Option, + #[arg(long)] + pub jaeger_addr: Option, + #[arg(long)] + pub socks_proxy: Option, + + // Output path + #[arg(long)] + pub guestos_config_json_path: PathBuf, +} + pub fn main() -> Result<()> { let opts = ConfigArgs::parse(); @@ -263,6 +337,45 @@ pub fn main() -> Result<()> { Ok(()) } - None => Ok(()), + Some(Commands::GenerateTestnetConfig(clap_args)) => { + // Convert `clap_args` into `GenerateTestnetConfigArgs` + let args = GenerateTestnetConfigArgs { + ipv6_config_type: clap_args.ipv6_config_type, + deterministic_prefix: clap_args.deterministic_prefix, + deterministic_prefix_length: clap_args.deterministic_prefix_length, + deterministic_gateway: clap_args.deterministic_gateway, + fixed_address: clap_args.fixed_address, + fixed_gateway: clap_args.fixed_gateway, + ipv4_address: clap_args.ipv4_address, + ipv4_gateway: clap_args.ipv4_gateway, + ipv4_prefix_length: clap_args.ipv4_prefix_length, + ipv4_domain: clap_args.ipv4_domain, + mgmt_mac: clap_args.mgmt_mac, + deployment_environment: clap_args.deployment_environment, + elasticsearch_hosts: clap_args.elasticsearch_hosts, + elasticsearch_tags: clap_args.elasticsearch_tags, + nns_public_key_path: clap_args.nns_public_key_path, + nns_urls: clap_args.nns_urls, + node_operator_private_key_path: clap_args.node_operator_private_key_path, + ssh_authorized_keys_path: clap_args.ssh_authorized_keys_path, + ic_crypto_path: clap_args.ic_crypto_path, + ic_state_path: clap_args.ic_state_path, + ic_registry_local_store_path: clap_args.ic_registry_local_store_path, + backup_retention_time_seconds: clap_args.backup_retention_time_seconds, + backup_purging_interval_seconds: clap_args.backup_purging_interval_seconds, + malicious_behavior: clap_args.malicious_behavior, + query_stats_epoch_length: clap_args.query_stats_epoch_length, + bitcoind_addr: clap_args.bitcoind_addr, + jaeger_addr: clap_args.jaeger_addr, + socks_proxy: clap_args.socks_proxy, + guestos_config_json_path: clap_args.guestos_config_json_path, + }; + + generate_testnet_config(args) + } + None => { + println!("No command provided. Use --help for usage information."); + Ok(()) + } } } From fa2ddfa73b93643a84c4422009fd0fc76979af8c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 21 Oct 2024 22:52:44 +0000 Subject: [PATCH 104/241] Fix default config values --- .../config/src/generate_testnet_config.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index ab3af2d6808..949711b1c8e 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -170,21 +170,14 @@ pub fn generate_testnet_config(args: GenerateTestnetConfigArgs) -> Result<()> { }; let nns_public_key_path = - nns_public_key_path.unwrap_or_else(|| PathBuf::from("/path/to/nns_public_key.pem")); + nns_public_key_path.unwrap_or_else(|| PathBuf::from("/boot/config/nns_public_key.pem")); let nns_urls = match nns_urls { - Some(urls) => { - let parsed_urls = urls - .iter() - .map(|s| Url::parse(s)) - .collect::, _>>()?; - if parsed_urls.is_empty() { - vec![Url::parse("http://localhost")?] - } else { - parsed_urls - } - } - None => vec![Url::parse("http://localhost")?], + Some(urls) => urls + .iter() + .map(|s| Url::parse(s)) + .collect::, _>>()?, + None => vec![Url::parse("https://wiki.internetcomputer.org")?], }; let icos_settings = ICOSSettings { From a9968f7364f6fd8f8da6e619be6f97c14f6a03aa Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 21 Oct 2024 23:13:30 +0000 Subject: [PATCH 105/241] Construct guestos config object from launch-single-vm --- Cargo.lock | 1 + .../launch-single-vm/BUILD.bazel | 1 + .../launch-single-vm/Cargo.toml | 1 + .../launch-single-vm/src/main.rs | 51 +++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index a979bb24675..aa23ae0eda3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14468,6 +14468,7 @@ name = "launch-single-vm" version = "0.1.0" dependencies = [ "clap 4.5.19", + "config", "ic-prep", "ic-registry-subnet-type", "ic-system-test-driver", diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel b/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel index 9032bdbc028..06cf195743b 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel +++ b/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel @@ -4,6 +4,7 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. + "//rs/ic_os/config:config_lib", "//rs/prep", "//rs/registry/subnet_type", "//rs/tests/driver:ic-system-test-driver", diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml b/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml index a77464636fb..8f798f809e6 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml +++ b/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml @@ -10,6 +10,7 @@ ic-prep = { path = "../../../prep" } ic-registry-subnet-type = { path = "../../../registry/subnet_type" } ic-system-test-driver = { path = "../../../tests/driver" } ic-types = { path = "../../../types/types" } +config = { path = "../../config" } clap = { workspace = true } reqwest = { workspace = true } diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index cd895d0320a..a0a6d26d409 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -23,6 +23,8 @@ use std::process::Command; use tempfile::tempdir; use url::Url; +use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; + const FARM_BASE_URL: &str = "https://farm.dfinity.systems"; /// Deploy a single ICOS VM to Farm @@ -200,6 +202,55 @@ fn main() { std::fs::copy(key, keys_dir.join("admin")).unwrap(); } + // Build GuestOS config object + let guestos_config_json_path = tempdir.as_ref().join("guestos_config.json"); + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some("RouterAdvertisement".to_string()), + deterministic_prefix: None, + deterministic_prefix_length: None, + deterministic_gateway: None, + fixed_address: None, + fixed_gateway: None, + ipv4_address: None, + ipv4_gateway: None, + ipv4_prefix_length: None, + ipv4_domain: None, + mgmt_mac: None, + deployment_environment: Some("testnet".to_string()), + elasticsearch_hosts: None, + elasticsearch_tags: None, + nns_public_key_path: None, + nns_urls: Some(vec![format!("http://[{}]", ipv6_addr)]), + node_operator_private_key_path: None, + ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), + ic_crypto_path: None, + ic_state_path: None, + ic_registry_local_store_path: None, + backup_retention_time_seconds: Some(3600), + backup_purging_interval_seconds: None, + malicious_behavior: None, + query_stats_epoch_length: None, + bitcoind_addr: None, + jaeger_addr: None, + socks_proxy: None, + guestos_config_json_path: guestos_config_json_path.clone(), + }; + + match generate_testnet_config(args) { + Ok(()) => { + let contents = std::fs::read_to_string(&guestos_config_json_path) + .expect("Failed to read the file"); + println!("{}", contents); + } + Err(e) => { + println!("Failed to generate testnet config: {:?}", e); + } + } + + std::thread::sleep(std::time::Duration::from_secs(10 * 6000)); + + // todo: pass guestos config object to build-bootstrap after rewriting the script to accept config object + // Build config image let filename = "config.tar.gz"; let config_path = tempdir.as_ref().join(filename); From e61ca1dea3d95fb4fd6c580f1ffcc0dea615d1fe Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 15:44:26 +0000 Subject: [PATCH 106/241] Pass guestos-config to build-bootstrap --- .../build-bootstrap-config-image.sh | 44 ++++++++----------- .../dev-generate-guestos-config.sh | 5 ++- .../generate-guestos-config.sh | 5 ++- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 4350eedbfd2..93cbe90597b 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -16,9 +16,8 @@ Following that are the options specifying the configuration to write. Each of option takes a value given as next argument, and any number of the following options may be specified: - --ipv6_address a:b::c/n - The IPv6 address to assign. Must include netmask in bits (e.g. - dead:beef::1/64). Overrides all other generation for testing. + --guestos_config path + The serialized GuestOS config object. --ic_crypto path Injected crypto state. Should point to a directory containing material @@ -56,10 +55,9 @@ function build_ic_bootstrap_tar() { local OUT_FILE="$1" shift - local IPV6_ADDRESS + local GUESTOS_CONFIG + local NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY ACCOUNTS_SSH_AUTHORIZED_KEYS local IC_CRYPTO IC_STATE IC_REGISTRY_LOCAL_STORE - local NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY - local ACCOUNTS_SSH_AUTHORIZED_KEYS while true; do if [ $# == 0 ]; then @@ -67,8 +65,17 @@ function build_ic_bootstrap_tar() { fi case "$1" in - --ipv6_address) - IPV6_ADDRESS="$2" + --guestos_config) + GUESTOS_CONFIG="$2" + ;; + --nns_public_key) + NNS_PUBLIC_KEY="$2" + ;; + --node_operator_private_key) + NODE_OPERATOR_PRIVATE_KEY="$2" + ;; + --accounts_ssh_authorized_keys) + ACCOUNTS_SSH_AUTHORIZED_KEYS="$2" ;; --ic_crypto) IC_CRYPTO="$2" @@ -79,15 +86,6 @@ function build_ic_bootstrap_tar() { --ic_registry_local_store) IC_REGISTRY_LOCAL_STORE="$2" ;; - --nns_public_key) - NNS_PUBLIC_KEY="$2" - ;; - --accounts_ssh_authorized_keys) - ACCOUNTS_SSH_AUTHORIZED_KEYS="$2" - ;; - --node_operator_private_key) - NODE_OPERATOR_PRIVATE_KEY="$2" - ;; *) echo "Unrecognized option: $1" usage @@ -100,7 +98,10 @@ function build_ic_bootstrap_tar() { local BOOTSTRAP_TMPDIR=$(mktemp -d) - # todo: switch nns_public_key.pem, node_operator_private_key.pem. and accounts_ssh to use config object + if [ "${GUESTOS_CONFIG}" != "" ]; then + cp "${GUESTOS_CONFIG}" "${BOOTSTRAP_TMPDIR}/config.json" + fi + if [ "${NNS_PUBLIC_KEY}" != "" ]; then cp "${NNS_PUBLIC_KEY}" "${BOOTSTRAP_TMPDIR}/nns_public_key.pem" fi @@ -111,7 +112,6 @@ function build_ic_bootstrap_tar() { cp -r "${ACCOUNTS_SSH_AUTHORIZED_KEYS}" "${BOOTSTRAP_TMPDIR}/accounts_ssh_authorized_keys" fi - # todo: investigate what to do for... if [ "${IC_CRYPTO}" != "" ]; then cp -r "${IC_CRYPTO}" "${BOOTSTRAP_TMPDIR}/ic_crypto" fi @@ -122,12 +122,6 @@ function build_ic_bootstrap_tar() { cp -r "${IC_REGISTRY_LOCAL_STORE}" "${BOOTSTRAP_TMPDIR}/ic_registry_local_store" fi - # Create guestos config.json - echo "* Generating 'config-guestos.json'..." - /opt/ic/bin/config generate-guestos-config --guestos-ipv6-address "$IPV6_ADDRESS" - echo "* Copying 'config-guestos.json' to GuestOS config partition..." - cp /boot/config/config-guestos.json "${BOOTSTRAP_TMPDIR}/config.json" - tar cf "${OUT_FILE}" \ --sort=name \ --owner=root:0 \ diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 449c1be27b8..d0445d76da3 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -65,8 +65,11 @@ function read_config_variables() { } function assemble_config_media() { + ipv6_address="$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)" + /opt/ic/bin/config generate-guestos-config --guestos-ipv6-address "$ipv6_address" + cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) - cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") + cmd+=(--guestos_config "/boot/config/config-guestos.json") cmd+=(--nns_public_key "$nns_public_key") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 2e2a9c581f4..3cf5292dfff 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -64,8 +64,11 @@ function read_config_variables() { } function assemble_config_media() { + ipv6_address="$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)" + /opt/ic/bin/config generate-guestos-config --guestos-ipv6-address "$ipv6_address" + cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) - cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") + cmd+=(--guestos_config "/boot/config/config-guestos.json") cmd+=(--nns_public_key "$nns_public_key") if [ -f "$node_operator_private_key" ]; then cmd+=(--node_operator_private_key "$node_operator_private_key") From 223e12c20a3888906dc02ad85337e9a78244682f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 17:39:46 +0000 Subject: [PATCH 107/241] Fix configure_ipv6 --- ic-os/components/ic/generate-replica-config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 887320581ee..27db1d984c8 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -27,7 +27,7 @@ function read_config_variables() { } function configure_ipv6() { - ipv6_config_type=$(get_config_value '.network_settings.ipv6_config | keys[]') + ipv6_config_type=$(get_config_value '.network_settings.ipv6_config | if type=="object" then keys[] else . end') case "$ipv6_config_type" in "Deterministic") echo "GuestOS IPv6 configuration should not be 'Deterministic'." @@ -46,7 +46,7 @@ function configure_ipv6() { ;; esac - if [ "${IPV6_ADDRESS}" == "" ]; then + if [ -z "${IPV6_ADDRESS}" ]; then echo "Cannot determine an IPv6 address, aborting" exit 1 fi From 67a8affa68bdd416b3033b9b3ac48c440ca93b65 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 17:40:05 +0000 Subject: [PATCH 108/241] Move launch-single-vm to use config_tool --- rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index a0a6d26d409..d518afc632e 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -226,7 +226,7 @@ fn main() { ic_crypto_path: None, ic_state_path: None, ic_registry_local_store_path: None, - backup_retention_time_seconds: Some(3600), + backup_retention_time_seconds: None, backup_purging_interval_seconds: None, malicious_behavior: None, query_stats_epoch_length: None, @@ -247,18 +247,14 @@ fn main() { } } - std::thread::sleep(std::time::Duration::from_secs(10 * 6000)); - - // todo: pass guestos config object to build-bootstrap after rewriting the script to accept config object - // Build config image let filename = "config.tar.gz"; let config_path = tempdir.as_ref().join(filename); let local_store = prep_dir.join("ic_registry_local_store"); Command::new(build_bootstrap_script) .arg(&config_path) - .arg("--nns_urls") - .arg(ipv6_addr.to_string()) + .arg("--guestos_config") + .arg(guestos_config_json_path) .arg("--ic_crypto") .arg(node.crypto_path()) .arg("--ic_registry_local_store") From 7bc2270da2f0edf49a1e83ca57f5887c6a8a1a55 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 19:30:19 +0000 Subject: [PATCH 109/241] Update config documentation --- ic-os/docs/Configuration.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index 5d41f53a135..9be971fa947 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -5,7 +5,7 @@ Each IC-OS has a 100 MB config partition. All IC-OS config partitions are initia In production, configuration is propagated from a partition on the USB installer through each of SetupOS, HostOS and GuestOS. This process is controlled by the (link:../../rs/ic_os/config/README.md[ic-os config tool]) and an assortment of bash scripts. -All access to the config partition should be done through the ic-os config tool. +All access to the config partition should be done through the ic-os config tool and config object. == User-facing configuration files From ea6a2ac18c62517464e0827480a2f88f22f19477 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 20:33:19 +0000 Subject: [PATCH 110/241] process_nns_urls --- .../components/ic/generate-replica-config.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 27db1d984c8..0deca084f15 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -112,6 +112,26 @@ function set_default_config_values() { NODE_INDEX="0" } +# If the URL is of the form "https://[IPv6]/" then we extract the IPv6 address. +# Otherwise, we copy over the whole URL to ic.json5 +# If a URL is of the form "http://[IPv6_address]/" or "https://[IPv6_address]/", +# it extracts just the IPv6 address. +# For all other URLs, it leaves them unchanged. +function process_nns_urls() { + local processed_urls=() + IFS=',' read -ra URLS <<< "$NNS_URLS" + for url in "${URLS[@]}"; do + if [[ $url =~ ^https?://\[([0-9a-fA-F:]+)\](:[0-9]+)?(/.*)?$ ]]; then + ipv6_addr="${BASH_REMATCH[1]}" + processed_urls+=("$ipv6_addr") + else + # Keep the URL as is + processed_urls+=("$url") + fi + done + NNS_URLS=$(IFS=','; echo "${processed_urls[*]}") +} + while getopts "i:o:" OPT; do case "${OPT}" in i) @@ -137,6 +157,7 @@ configure_ipv4 read_config_variables set_default_config_values +process_nns_urls sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \ From 477fcee63f75488fac4dccffc003653058c2c5f6 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 20:58:38 +0000 Subject: [PATCH 111/241] Update == HostOS -> GuestOS config documentation --- ic-os/docs/Configuration.adoc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index 9be971fa947..325f4c81324 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -20,11 +20,10 @@ Refer to link:../../rs/ic_os/config/README.md[rs/ic_os/config] and link:../compo == HostOS -> GuestOS -TODO: update... +HostOS creates a bootstrap config image containing a tar file with the GuestOS configuration files. -HostOS builds the "bootstrap config image". Refer to link:../components/hostos-scripts/build-bootstrap-config-image.sh[build-bootstrap-config-image.sh] +Refer to link:../components/hostos-scripts/generate-guestos-config.sh[generate-guestos-config.sh] and link:../components/hostos-scripts/build-bootstrap-config-image.sh[build-bootstrap-config-image.sh] for more details. -The bootstrap config image contains a tar file with the GuestOS configuration files. When the HostOS launches the GuestOS, the bootstrap config image is attached to the GuestOS as a virtual USB. Refer to link:../components/hostos-scripts/guestos/guestos.xml.template[guestos.xml.template] When the GuestOS boots, it checks for available removable media devices (i.e. the bootstrap config image). If such a device is found, the media must contain a VFAT filesystem and a single file called `ic-bootstrap.tar`. From b5432c3d3fe195e5be5612bbea123f147f6e2d82 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 20:59:09 +0000 Subject: [PATCH 112/241] Fix filebeat service --- ic-os/components/monitoring/filebeat/filebeat.service | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ic-os/components/monitoring/filebeat/filebeat.service b/ic-os/components/monitoring/filebeat/filebeat.service index 709b3f64864..8707e255810 100644 --- a/ic-os/components/monitoring/filebeat/filebeat.service +++ b/ic-os/components/monitoring/filebeat/filebeat.service @@ -17,6 +17,10 @@ Group=filebeat Environment="GODEBUG='madvdontneed=1'" ExecStartPre=+/opt/ic/bin/setup-filebeat-permissions.sh ExecStartPre=+/opt/ic/bin/generate-filebeat-config.sh -i /etc/filebeat/filebeat.yml.template -o /run/ic-node/etc/filebeat/filebeat.yml + +# Only start Filebeat if configuration file is generated +ExecCondition=/usr/bin/test -f /run/ic-node/etc/filebeat/filebeat.yml + ExecStart=/usr/local/bin/filebeat --environment systemd -e --path.home /var/lib/filebeat --path.config /run/ic-node/etc/filebeat --path.data /var/lib/filebeat --path.logs /var/log/filebeat Restart=always From 11e5ac2bb0f08a73877ad3edc240cd65f011379c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 21:57:58 +0000 Subject: [PATCH 113/241] Fix MALICIOUS_BEHAVIOR generate-replica-config --- ic-os/components/ic/generate-replica-config.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 0deca084f15..ed134469665 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -23,7 +23,9 @@ function read_config_variables() { BACKUP_PURGING_INTERVAL_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_purging_interval_seconds') QUERY_STATS_EPOCH_LENGTH=$(get_config_value '.guestos_settings.guestos_dev_settings.query_stats_epoch_length') JAEGER_ADDR=$(get_config_value '.guestos_settings.guestos_dev_settings.jaeger_addr') - MALICIOUS_BEHAVIOR=$(get_config_value '.guestos_settings.guestos_dev_settings.malicious_behavior') + + # Compact the JSON and escape special characters + MALICIOUS_BEHAVIOR=$(get_config_value '.guestos_settings.guestos_dev_settings.malicious_behavior' | jq -c '.' | sed 's/[&\/]/\\&/g') } function configure_ipv6() { From 02a3c3a8b5f08cd18b0d8ddbb96c7f5fe5f4fd29 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 22:00:35 +0000 Subject: [PATCH 114/241] Remove unnecessary TODO --- ic-os/docs/Configuration.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index 325f4c81324..a62902136b3 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -104,8 +104,6 @@ After all is done, it is advised to prepare a configuration for a single node an === Injecting external state -TODO: update... - *Typical bootstrap process:* On first boot, the system will perform technical initialization (filesystems, etc.) and afterwards, initialize itself to act as a node in the IC. The node is initialized using key generation on the node itself (such that the private key never leaves the node) and through joining the IC (the node gets the rest of its state via joining the IC). "Registration" to the target IC is initiated by the node itself by sending a Node Operator-signed "join" request to its NNS. However, the typical bootstrap process can be modified such that the node is initialized using externally generated private keys and an externally generated initial state. All "registration" to the target IC is assumed to have been performed by other means. From 6ccef2580e6b0c08e414f2bb7789ff46b874a9f4 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 22:04:02 +0000 Subject: [PATCH 115/241] Remove testing logic and clean up launch-single-vm --- .../dev_test_tools/launch-single-vm/src/main.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index d518afc632e..0177f80b769 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -24,6 +24,7 @@ use tempfile::tempdir; use url::Url; use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; +use ic_types::malicious_behaviour::MaliciousBehaviour; const FARM_BASE_URL: &str = "https://farm.dfinity.systems"; @@ -236,16 +237,8 @@ fn main() { guestos_config_json_path: guestos_config_json_path.clone(), }; - match generate_testnet_config(args) { - Ok(()) => { - let contents = std::fs::read_to_string(&guestos_config_json_path) - .expect("Failed to read the file"); - println!("{}", contents); - } - Err(e) => { - println!("Failed to generate testnet config: {:?}", e); - } - } + // populate guestos_config_json_path with serialized guestos config object + let _ = generate_testnet_config(args); // Build config image let filename = "config.tar.gz"; From 07156b301c49307c3d1e932b7c921b53a572fbd1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 22 Oct 2024 22:35:51 +0000 Subject: [PATCH 116/241] Remove unused import --- rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index 0177f80b769..8f9f20963e5 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -24,7 +24,6 @@ use tempfile::tempdir; use url::Url; use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; -use ic_types::malicious_behaviour::MaliciousBehaviour; const FARM_BASE_URL: &str = "https://farm.dfinity.systems"; From c542e263fbfe26f03087b66a05d7e0584c913192 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 15:45:01 +0000 Subject: [PATCH 117/241] Update Testing configuration documentation --- ic-os/docs/Configuration.adoc | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index a62902136b3..70b1f2a6b29 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -89,15 +89,12 @@ Consider that values may be controlled by an attacker on boot. Bootstrapping a n === Testing -TODO: update... - -* *bootstrap-ic-node.sh* can be temporarily tweaked (internally adapt paths, then run the process_bootstrap function): -** run stand-alone -** verify that the config image is unpacked -** verify its files are in the correct locations +For testing, to add new configuration bits, you can modify the config tool located at +link:../../rs/ic_os/config/README.md[rs/ic_os/config]. Or, you may find it easier to update *bootstrap-ic-node.sh* directly, +particularly if you wish to add a new configuration file (as opposed to just a new configuration _field_). +* *ic_os config tool* can be run stand-alone to verify that it produces the intended configuration object. * *bootstrap-ic-node.sh* can be temporarily tweaked (internally adapt paths; basically just running the process_bootstrap function of it) to run stand-alone and verify that the config image is unpacked and its files sorted into the correct locations. - * *generate-replica-config.sh* can be run stand-alone to verify that it produces the intended ic.json5 configuration from the template. After all is done, it is advised to prepare a configuration for a single node and boot it in a VM before conducting testnet deployments. From 20db7ac93c0f5e71d6d97864c1b075695542cd90 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 16:00:03 +0000 Subject: [PATCH 118/241] Fix IPV6_ADDRESS generate-replica-config --- ic-os/components/ic/generate-replica-config.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index ed134469665..b8cf6b292ff 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -37,6 +37,8 @@ function configure_ipv6() { ;; "Fixed") IPV6_ADDRESS=$(get_config_value '.network_settings.ipv6_config.Fixed.address') + # Remove the subnet part from the IPv6 address + IPV6_ADDRESS="${IPV6_ADDRESS%%/*}" ;; "RouterAdvertisement") interface=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';')) From b138af6b2a393abcd5a116ca7e52e5771b5b8179 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 20:37:10 +0000 Subject: [PATCH 119/241] Update farm to use new config tool --- Cargo.lock | 1 + rs/tests/driver/BUILD.bazel | 1 + rs/tests/driver/Cargo.toml | 1 + rs/tests/driver/src/driver/bootstrap.rs | 119 +++++++++++++++--------- 4 files changed, 76 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 36ba9d1be96..a729f8a4569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12401,6 +12401,7 @@ dependencies = [ "canister-test", "chrono", "clap 4.5.19", + "config", "crossbeam-channel", "cycles-minting-canister", "deterministic_ips", diff --git a/rs/tests/driver/BUILD.bazel b/rs/tests/driver/BUILD.bazel index d276b82a20b..19c8c3ee99f 100644 --- a/rs/tests/driver/BUILD.bazel +++ b/rs/tests/driver/BUILD.bazel @@ -47,6 +47,7 @@ rust_library( "//rs/crypto/tree_hash", "//rs/crypto/utils/threshold_sig_der", "//rs/cycles_account_manager", + "//rs/ic_os/config:config_lib", "//rs/ic_os/dev_test_tools/deterministic_ips", "//rs/interfaces", "//rs/interfaces/registry", diff --git a/rs/tests/driver/Cargo.toml b/rs/tests/driver/Cargo.toml index 6633f3082dd..f24636b3605 100644 --- a/rs/tests/driver/Cargo.toml +++ b/rs/tests/driver/Cargo.toml @@ -17,6 +17,7 @@ candid = { workspace = true } canister-test = { path = "../../rust_canisters/canister_test" } chrono = { workspace = true } clap = { workspace = true } +config = { path = "../../ic_os/config" } crossbeam-channel = { workspace = true } cycles-minting-canister = { path = "../../nns/cmc" } deterministic_ips = { path = "../../ic_os/dev_test_tools/deterministic_ips" } diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index d3e154617c9..63334f35106 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -22,6 +22,7 @@ use crate::k8s::images::*; use crate::k8s::tnet::{TNet, TNode}; use crate::util::block_on; use anyhow::{bail, Result}; +use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; use ic_base_types::NodeId; use ic_prep_lib::{ internet_computer::{IcConfig, InitializedIc, TopologyConfig}, @@ -410,33 +411,44 @@ fn create_config_disk_image( test_env: &TestEnv, group_name: &str, ) -> anyhow::Result<()> { - let img_path = PathBuf::from(&node.node_path).join(CONF_IMG_FNAME); - let script_path = - get_dependency_path("ic-os/components/hostos-scripts/build-bootstrap-config-image.sh"); - let mut cmd = Command::new(script_path); - let local_store_path = test_env - .prep_dir(ic_name) - .expect("no no-name IC") - .registry_local_store_path(); - cmd.arg(img_path.clone()) - .arg("--hostname") - .arg(node.node_id.to_string()) - .arg("--ic_registry_local_store") - .arg(local_store_path) - .arg("--ic_state") - .arg(node.state_path()) - .arg("--ic_crypto") - .arg(node.crypto_path()) - .arg("--elasticsearch_tags") - .arg(format!("system_test {}", group_name)); + let mut args = GenerateTestnetConfigArgs { + ipv6_config_type: Some("RouterAdvertisement".to_string()), + deterministic_prefix: None, + deterministic_prefix_length: None, + deterministic_gateway: None, + fixed_address: None, + fixed_gateway: None, + ipv4_address: None, + ipv4_gateway: None, + ipv4_prefix_length: None, + ipv4_domain: None, + mgmt_mac: None, + deployment_environment: Some("testnet".to_string()), + elasticsearch_hosts: None, + elasticsearch_tags: Some(format!("system_test {}", group_name)), + nns_public_key_path: None, + nns_urls: None, + node_operator_private_key_path: None, + ssh_authorized_keys_path: None, + ic_crypto_path: None, + ic_state_path: None, + ic_registry_local_store_path: None, + backup_retention_time_seconds: None, + backup_purging_interval_seconds: None, + malicious_behavior: None, + query_stats_epoch_length: None, + bitcoind_addr: None, + jaeger_addr: None, + socks_proxy: None, + guestos_config_json_path: guestos_config_json_path.clone(), + }; // We've seen k8s nodes fail to pick up RA correctly, so we specify their // addresses directly. Ideally, all nodes should do this, to match mainnet. if InfraProvider::read_attribute(test_env) == InfraProvider::K8s { - cmd.arg("--ipv6_address") - .arg(format!("{}/64", node.node_config.public_api.ip())) - .arg("--ipv6_gateway") - .arg("fe80::ecee:eeff:feee:eeee"); + args.ipv6_config_type = Some("Fixed".to_string()); + args.fixed_address = Some(format!("{}/64", node.node_config.public_api.ip())); + args.fixed_gateway = Some("fe80::ecee:eeff:feee:eeee".to_string()); } // If we have a root subnet, specify the correct NNS url. @@ -446,8 +458,7 @@ fn create_config_disk_image( .nodes() .next() { - cmd.arg("--nns_urls") - .arg(format!("http://[{}]:8080", node.get_ip_addr())); + args.nns_urls = Some(format!("http://[{}]:8080", node.get_ip_addr())); } if let Some(malicious_behavior) = malicious_behavior { @@ -455,8 +466,7 @@ fn create_config_disk_image( test_env.logger(), "Node with id={} has malicious behavior={:?}", node.node_id, malicious_behavior ); - cmd.arg("--malicious_behavior") - .arg(serde_json::to_string(&malicious_behavior)?); + args.malicious_behavior = Some(serde_json::to_string(&malicious_behavior)?); } if let Some(query_stats_epoch_length) = query_stats_epoch_length { @@ -466,8 +476,7 @@ fn create_config_disk_image( node.node_id, query_stats_epoch_length ); - cmd.arg("--query_stats_epoch_length") - .arg(format!("{}", query_stats_epoch_length)); + args.query_stats_epoch_length = Some(format!("{}", query_stats_epoch_length)); } if let Some(ipv4_config) = ipv4_config { @@ -475,12 +484,9 @@ fn create_config_disk_image( test_env.logger(), "Node with id={} is IPv4-enabled: {:?}", node.node_id, ipv4_config ); - cmd.arg("--ipv4_address").arg(format!( - "{}/{:?}", - ipv4_config.ip_addr(), - ipv4_config.prefix_length() - )); - cmd.arg("--ipv4_gateway").arg(ipv4_config.gateway_ip_addr()); + args.ipv4_address = Some(ipv4_config.ip_addr().to_string()); + args.ipv4_gateway = Some(ipv4_config.gateway_ip_addr().to_string()); + args.ipv4_prefix_length = Some(ipv4_config.prefix_length().to_string()); } if let Some(domain) = domain { @@ -488,13 +494,12 @@ fn create_config_disk_image( test_env.logger(), "Node with id={} has domain_name {}", node.node_id, domain, ); - cmd.arg("--domain").arg(domain); + args.domain = Some(domain); } let ssh_authorized_pub_keys_dir: PathBuf = test_env.get_path(SSH_AUTHORIZED_PUB_KEYS_DIR); if ssh_authorized_pub_keys_dir.exists() { - cmd.arg("--accounts_ssh_authorized_keys") - .arg(ssh_authorized_pub_keys_dir); + args.ssh_authorized_keys_path = Some(ssh_authorized_pub_keys_dir); } let elasticsearch_hosts: Vec = get_elasticsearch_hosts()?; @@ -503,22 +508,44 @@ fn create_config_disk_image( "ElasticSearch hosts are {:?}", elasticsearch_hosts ); if !elasticsearch_hosts.is_empty() { - cmd.arg("--elasticsearch_hosts") - .arg(elasticsearch_hosts.join(" ")); + args.elasticsearch_hosts = Some(elasticsearch_hosts.join(" ")); } // --bitcoind_addr indicates the local bitcoin node that the bitcoin adapter should be connected to in the system test environment. - if let Ok(arg) = test_env.read_json_object::(BITCOIND_ADDR_PATH) { - cmd.arg("--bitcoind_addr").arg(arg); + if let Ok(bitcoin_addr) = test_env.read_json_object::(BITCOIND_ADDR_PATH) { + args.bitcoind_addr = Some(bitcoin_addr); } + // --jaeger_addr indicates the local Jaeger node that the nodes should be connected to in the system test environment. - if let Ok(arg) = test_env.read_json_object::(JAEGER_ADDR_PATH) { - cmd.arg("--jaeger_addr").arg(arg); + if let Ok(jaeger_addr) = test_env.read_json_object::(JAEGER_ADDR_PATH) { + args.jaeger_addr = Some(jaeger_addr); } + // --socks_proxy indicates that a socks proxy is available to the system test environment. - if let Ok(arg) = test_env.read_json_object::(SOCKS_PROXY_PATH) { - cmd.arg("--socks_proxy").arg(arg); + if let Ok(socks_proxy) = test_env.read_json_object::(SOCKS_PROXY_PATH) { + args.socks_proxy = Some(socks_proxy); } + + // populate guestos_config_json_path with serialized guestos config object + let _ = generate_testnet_config(args); + + let img_path = PathBuf::from(&node.node_path).join(CONF_IMG_FNAME); + let script_path = + get_dependency_path("ic-os/components/hostos-scripts/build-bootstrap-config-image.sh"); + let mut cmd = Command::new(script_path); + let local_store_path = test_env + .prep_dir(ic_name) + .expect("no no-name IC") + .registry_local_store_path(); + + cmd.arg(img_path.clone()) + .arg("--ic_registry_local_store") + .arg(local_store_path) + .arg("--ic_state") + .arg(node.state_path()) + .arg("--ic_crypto") + .arg(node.crypto_path()); + let key = "PATH"; let old_path = match std::env::var(key) { Ok(val) => { From 63c199b21d043767cc7444638d5d9b548728ce4c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 20:40:08 +0000 Subject: [PATCH 120/241] Update call to generate_testnet_config --- rs/tests/driver/src/driver/bootstrap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 63334f35106..aea48b7db79 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -527,7 +527,7 @@ fn create_config_disk_image( } // populate guestos_config_json_path with serialized guestos config object - let _ = generate_testnet_config(args); + generate_testnet_config(args)?; let img_path = PathBuf::from(&node.node_path).join(CONF_IMG_FNAME); let script_path = From ceb67240c76aa60603b48bb11ec49794d46fa811 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Wed, 23 Oct 2024 20:44:52 +0000 Subject: [PATCH 121/241] Automatically updated Cargo*.lock --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index b641da059e4..85a79431669 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12429,6 +12429,7 @@ dependencies = [ "canister-test", "chrono", "clap 4.5.20", + "config", "crossbeam-channel", "cycles-minting-canister", "deterministic_ips", @@ -14506,6 +14507,7 @@ name = "launch-single-vm" version = "0.1.0" dependencies = [ "clap 4.5.20", + "config", "ic-prep", "ic-registry-subnet-type", "ic-system-test-driver", From 0e7d62de5d4e87d9e28dec6b20a464e84c88bf92 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 20:48:12 +0000 Subject: [PATCH 122/241] Fix config library visibility --- rs/ic_os/config/BUILD.bazel | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rs/ic_os/config/BUILD.bazel b/rs/ic_os/config/BUILD.bazel index 5e4ed19f5dd..0e5281a4897 100644 --- a/rs/ic_os/config/BUILD.bazel +++ b/rs/ic_os/config/BUILD.bazel @@ -32,6 +32,10 @@ rust_library( ["src/**/*.rs"], exclude = ["src/main.rs"], ), + visibility = [ + "//rs:ic-os-pkg", + "//rs:system-tests-pkg", + ], crate_name = "config", edition = "2021", deps = DEPENDENCIES, From 5c67331ceccfbdab863fac01e06942225be41556 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 20:51:17 +0000 Subject: [PATCH 123/241] Fix call to build-bootstrap-config-image --- rs/tests/driver/src/driver/bootstrap.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index aea48b7db79..251bc8c677d 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -411,6 +411,8 @@ fn create_config_disk_image( test_env: &TestEnv, group_name: &str, ) -> anyhow::Result<()> { + // Build GuestOS config object + let guestos_config_json_path = tempdir.as_ref().join("guestos_config.json"); let mut args = GenerateTestnetConfigArgs { ipv6_config_type: Some("RouterAdvertisement".to_string()), deterministic_prefix: None, @@ -539,6 +541,8 @@ fn create_config_disk_image( .registry_local_store_path(); cmd.arg(img_path.clone()) + .arg("--guestos_config") + .arg(guestos_config_json_path) .arg("--ic_registry_local_store") .arg(local_store_path) .arg("--ic_state") From 5da81fc170a7f6fdc4e60dbaed219cf8d7cf1862 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 20:58:36 +0000 Subject: [PATCH 124/241] Fix config construction --- rs/tests/driver/src/driver/bootstrap.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 251bc8c677d..cf26e89d524 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -46,6 +46,7 @@ use std::{ process::Command, thread::{self, JoinHandle}, }; +use tempfile::tempdir; use url::Url; use zstd::stream::write::Encoder; @@ -412,7 +413,7 @@ fn create_config_disk_image( group_name: &str, ) -> anyhow::Result<()> { // Build GuestOS config object - let guestos_config_json_path = tempdir.as_ref().join("guestos_config.json"); + let guestos_config_json_path = tempdir().unwrap().as_ref().join("guestos_config.json"); let mut args = GenerateTestnetConfigArgs { ipv6_config_type: Some("RouterAdvertisement".to_string()), deterministic_prefix: None, @@ -460,7 +461,7 @@ fn create_config_disk_image( .nodes() .next() { - args.nns_urls = Some(format!("http://[{}]:8080", node.get_ip_addr())); + args.nns_urls = Some(vec![format!("http://[{}]:8080", node.get_ip_addr())]); } if let Some(malicious_behavior) = malicious_behavior { @@ -478,7 +479,7 @@ fn create_config_disk_image( node.node_id, query_stats_epoch_length ); - args.query_stats_epoch_length = Some(format!("{}", query_stats_epoch_length)); + args.query_stats_epoch_length = Some(query_stats_epoch_length); } if let Some(ipv4_config) = ipv4_config { @@ -488,7 +489,7 @@ fn create_config_disk_image( ); args.ipv4_address = Some(ipv4_config.ip_addr().to_string()); args.ipv4_gateway = Some(ipv4_config.gateway_ip_addr().to_string()); - args.ipv4_prefix_length = Some(ipv4_config.prefix_length().to_string()); + args.ipv4_prefix_length = Some(ipv4_config.prefix_length().try_into().unwrap()); } if let Some(domain) = domain { @@ -496,7 +497,7 @@ fn create_config_disk_image( test_env.logger(), "Node with id={} has domain_name {}", node.node_id, domain, ); - args.domain = Some(domain); + args.ipv4_domain = Some(domain); } let ssh_authorized_pub_keys_dir: PathBuf = test_env.get_path(SSH_AUTHORIZED_PUB_KEYS_DIR); From af7ce6c5a009382db287a53bd02e86b61a556e56 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 21:07:13 +0000 Subject: [PATCH 125/241] Fix pre-commit and buildifier --- .../early-boot/setup-hostname/setup-hostname.sh | 2 +- ic-os/components/ic/generate-replica-config.sh | 15 +++++++++------ ic-os/components/setupos.bzl | 2 +- rs/ic_os/config/BUILD.bazel | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh index 0f4c7ab10cd..e39406d4887 100755 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh @@ -50,7 +50,7 @@ function validate_arguments() { function read_config_variables() { mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') - mgmt_mac=${mgmt_mac//:/} # Remove colons from mgmt_mac + mgmt_mac=${mgmt_mac//:/} # Remove colons from mgmt_mac } function construct_hostname() { diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index b8cf6b292ff..62c47c79a9b 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -107,9 +107,9 @@ function get_if_address_retries() { function set_default_config_values() { [ "${NNS_URLS}" = "null" ] && NNS_URLS="http://[::1]:8080" - [ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default value is 24h - [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default value is 1h - [ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # Default is 600 blocks (around 10min) + [ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default value is 24h + [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default value is 1h + [ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # Default is 600 blocks (around 10min) [ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" # todo: remove node_index variable and hard-code into ic.json5.template @@ -118,12 +118,12 @@ function set_default_config_values() { # If the URL is of the form "https://[IPv6]/" then we extract the IPv6 address. # Otherwise, we copy over the whole URL to ic.json5 -# If a URL is of the form "http://[IPv6_address]/" or "https://[IPv6_address]/", +# If a URL is of the form "http://[IPv6_address]/" or "https://[IPv6_address]/", # it extracts just the IPv6 address. # For all other URLs, it leaves them unchanged. function process_nns_urls() { local processed_urls=() - IFS=',' read -ra URLS <<< "$NNS_URLS" + IFS=',' read -ra URLS <<<"$NNS_URLS" for url in "${URLS[@]}"; do if [[ $url =~ ^https?://\[([0-9a-fA-F:]+)\](:[0-9]+)?(/.*)?$ ]]; then ipv6_addr="${BASH_REMATCH[1]}" @@ -133,7 +133,10 @@ function process_nns_urls() { processed_urls+=("$url") fi done - NNS_URLS=$(IFS=','; echo "${processed_urls[*]}") + NNS_URLS=$( + IFS=',' + echo "${processed_urls[*]}" + ) } while getopts "i:o:" OPT; do diff --git a/ic-os/components/setupos.bzl b/ic-os/components/setupos.bzl index 5d81395e835..11249c631e9 100644 --- a/ic-os/components/setupos.bzl +++ b/ic-os/components/setupos.bzl @@ -34,7 +34,7 @@ component_files = { Label("misc/config/setupos/config.sh"): "/opt/ic/bin/config.sh", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", Label("misc/chrony/chrony-var.service"): "/etc/systemd/system/chrony-var.service", - Label("misc/serial-getty@/setupos/override.conf"): "/etc/systemd/system/serial-getty@.service.d/override.conf", + Label("misc/serial-getty@/setupos/override.conf"): "/etc/systemd/system/serial-getty@.service.d/override.conf", Label("monitoring/journald.conf"): "/etc/systemd/journald.conf", # networking diff --git a/rs/ic_os/config/BUILD.bazel b/rs/ic_os/config/BUILD.bazel index 0e5281a4897..e157ba36a38 100644 --- a/rs/ic_os/config/BUILD.bazel +++ b/rs/ic_os/config/BUILD.bazel @@ -32,12 +32,12 @@ rust_library( ["src/**/*.rs"], exclude = ["src/main.rs"], ), + crate_name = "config", + edition = "2021", visibility = [ "//rs:ic-os-pkg", "//rs:system-tests-pkg", ], - crate_name = "config", - edition = "2021", deps = DEPENDENCIES, ) From 6e5969dbc09a234b6e207925767ad6f643b217b9 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 23 Oct 2024 22:00:24 +0000 Subject: [PATCH 126/241] Fix ssh access --- rs/tests/driver/src/driver/bootstrap.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index cf26e89d524..6e438323598 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -432,7 +432,7 @@ fn create_config_disk_image( nns_public_key_path: None, nns_urls: None, node_operator_private_key_path: None, - ssh_authorized_keys_path: None, + ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), ic_crypto_path: None, ic_state_path: None, ic_registry_local_store_path: None, @@ -500,11 +500,6 @@ fn create_config_disk_image( args.ipv4_domain = Some(domain); } - let ssh_authorized_pub_keys_dir: PathBuf = test_env.get_path(SSH_AUTHORIZED_PUB_KEYS_DIR); - if ssh_authorized_pub_keys_dir.exists() { - args.ssh_authorized_keys_path = Some(ssh_authorized_pub_keys_dir); - } - let elasticsearch_hosts: Vec = get_elasticsearch_hosts()?; info!( test_env.logger(), @@ -551,6 +546,12 @@ fn create_config_disk_image( .arg("--ic_crypto") .arg(node.crypto_path()); + let ssh_authorized_pub_keys_dir: PathBuf = test_env.get_path(SSH_AUTHORIZED_PUB_KEYS_DIR); + if ssh_authorized_pub_keys_dir.exists() { + cmd.arg("--accounts_ssh_authorized_keys") + .arg(ssh_authorized_pub_keys_dir); + } + let key = "PATH"; let old_path = match std::env::var(key) { Ok(val) => { From 1b92686c5830f01565a42af57f32d3e856cd2b4f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 24 Oct 2024 16:27:27 +0000 Subject: [PATCH 127/241] Remove unnecessary process_nns_urls function --- .../components/ic/generate-replica-config.sh | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 62c47c79a9b..df3309c7cb0 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -116,29 +116,6 @@ function set_default_config_values() { NODE_INDEX="0" } -# If the URL is of the form "https://[IPv6]/" then we extract the IPv6 address. -# Otherwise, we copy over the whole URL to ic.json5 -# If a URL is of the form "http://[IPv6_address]/" or "https://[IPv6_address]/", -# it extracts just the IPv6 address. -# For all other URLs, it leaves them unchanged. -function process_nns_urls() { - local processed_urls=() - IFS=',' read -ra URLS <<<"$NNS_URLS" - for url in "${URLS[@]}"; do - if [[ $url =~ ^https?://\[([0-9a-fA-F:]+)\](:[0-9]+)?(/.*)?$ ]]; then - ipv6_addr="${BASH_REMATCH[1]}" - processed_urls+=("$ipv6_addr") - else - # Keep the URL as is - processed_urls+=("$url") - fi - done - NNS_URLS=$( - IFS=',' - echo "${processed_urls[*]}" - ) -} - while getopts "i:o:" OPT; do case "${OPT}" in i) @@ -164,14 +141,12 @@ configure_ipv4 read_config_variables set_default_config_values -process_nns_urls sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \ -e "s@{{ ipv4_gateway }}@${IPV4_GATEWAY}@" \ -e "s@{{ domain }}@${DOMAIN}@" \ -e "s@{{ nns_urls }}@${NNS_URLS}@" \ - -e "s@{{ nns_urls }}@${NNS_URLS}@" \ -e "s@{{ node_index }}@${NODE_INDEX}@" \ -e "s@{{ backup_retention_time_secs }}@${BACKUP_RETENTION_TIME_SECS}@" \ -e "s@{{ backup_purging_interval_secs }}@${BACKUP_PURGING_INTERVAL_SECS}@" \ From a6810fb435bd0062d4e8a946fa1b0074d216dc29 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 24 Oct 2024 19:06:10 +0000 Subject: [PATCH 128/241] Clean up comments --- ic-os/components/ic/generate-replica-config.sh | 4 ++-- rs/ic_os/config/src/generate_testnet_config.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index df3309c7cb0..06447b51539 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -107,8 +107,8 @@ function get_if_address_retries() { function set_default_config_values() { [ "${NNS_URLS}" = "null" ] && NNS_URLS="http://[::1]:8080" - [ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default value is 24h - [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default value is 1h + [ "${BACKUP_RETENTION_TIME_SECS}" = "null" ] && BACKUP_RETENTION_TIME_SECS="86400" # Default is 24h + [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default is 1h [ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # Default is 600 blocks (around 10min) [ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 949711b1c8e..0a6201713b5 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -234,7 +234,6 @@ pub fn generate_testnet_config(args: GenerateTestnetConfigArgs) -> Result<()> { println!("GuestOSConfig: {:?}", guestos_config); - // Write the configuration to a file serialize_and_write_config(&guestos_config_json_path, &guestos_config)?; println!( From 5f6713470823629b770c8285cf877f0ed3a4b92e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 25 Oct 2024 21:46:51 +0000 Subject: [PATCH 129/241] Add GenerateGuestosConfig comment --- rs/ic_os/config/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 011a12d37b0..5c05661aa4a 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -305,6 +305,8 @@ pub fn main() -> Result<()> { *path = guestos_config_path.join("accounts_ssh_authorized_keys"); } + // TODO: We won't have to modify networking between the hostos and + // guestos config after completing the networking revamp (NODE-1327) let mut guestos_network_settings = hostos_config.network_settings; // Update the GuestOS networking if `guestos_ipv6_address` is provided match &guestos_network_settings.ipv6_config { From 8bf4e30f063f13558fab8ee9d1f5c09497287c7b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 25 Oct 2024 21:52:21 +0000 Subject: [PATCH 130/241] Move guestos_config_json_path to be a separate parameter --- .../config/src/generate_testnet_config.rs | 11 +++--- rs/ic_os/config/src/main.rs | 3 +- .../launch-single-vm/src/main.rs | 7 ++-- rs/tests/driver/src/driver/bootstrap.rs | 35 +++++++++---------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 0a6201713b5..6ba292e3245 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -43,14 +43,14 @@ pub struct GenerateTestnetConfigArgs { pub bitcoind_addr: Option, pub jaeger_addr: Option, pub socks_proxy: Option, - - // Output path - pub guestos_config_json_path: PathBuf, } /// Generates a writes a serialized GuestOSConfig to guestos_config_json_path /// Any required config fields that aren't specified will receive dummy values -pub fn generate_testnet_config(args: GenerateTestnetConfigArgs) -> Result<()> { +pub fn generate_testnet_config( + config: GenerateTestnetConfigArgs, + guestos_config_json_path: PathBuf, +) -> Result<()> { let GenerateTestnetConfigArgs { ipv6_config_type, deterministic_prefix, @@ -80,8 +80,7 @@ pub fn generate_testnet_config(args: GenerateTestnetConfigArgs) -> Result<()> { bitcoind_addr, jaeger_addr, socks_proxy, - guestos_config_json_path, - } = args; + } = config; // Construct the NetworkSettings let ipv6_config = match ipv6_config_type.as_deref() { diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 5c05661aa4a..722b310d93d 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -370,10 +370,9 @@ pub fn main() -> Result<()> { bitcoind_addr: clap_args.bitcoind_addr, jaeger_addr: clap_args.jaeger_addr, socks_proxy: clap_args.socks_proxy, - guestos_config_json_path: clap_args.guestos_config_json_path, }; - generate_testnet_config(args) + generate_testnet_config(args, clap_args.guestos_config_json_path) } None => { println!("No command provided. Use --help for usage information."); diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index 8f9f20963e5..1b0f4e64fc1 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -203,8 +203,7 @@ fn main() { } // Build GuestOS config object - let guestos_config_json_path = tempdir.as_ref().join("guestos_config.json"); - let args = GenerateTestnetConfigArgs { + let config = GenerateTestnetConfigArgs { ipv6_config_type: Some("RouterAdvertisement".to_string()), deterministic_prefix: None, deterministic_prefix_length: None, @@ -233,11 +232,11 @@ fn main() { bitcoind_addr: None, jaeger_addr: None, socks_proxy: None, - guestos_config_json_path: guestos_config_json_path.clone(), }; // populate guestos_config_json_path with serialized guestos config object - let _ = generate_testnet_config(args); + let guestos_config_json_path = tempdir.as_ref().join("guestos_config.json"); + let _ = generate_testnet_config(config, guestos_config_json_path); // Build config image let filename = "config.tar.gz"; diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 6e438323598..f55f5f2c562 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -413,8 +413,7 @@ fn create_config_disk_image( group_name: &str, ) -> anyhow::Result<()> { // Build GuestOS config object - let guestos_config_json_path = tempdir().unwrap().as_ref().join("guestos_config.json"); - let mut args = GenerateTestnetConfigArgs { + let mut config = GenerateTestnetConfigArgs { ipv6_config_type: Some("RouterAdvertisement".to_string()), deterministic_prefix: None, deterministic_prefix_length: None, @@ -443,15 +442,14 @@ fn create_config_disk_image( bitcoind_addr: None, jaeger_addr: None, socks_proxy: None, - guestos_config_json_path: guestos_config_json_path.clone(), }; // We've seen k8s nodes fail to pick up RA correctly, so we specify their // addresses directly. Ideally, all nodes should do this, to match mainnet. if InfraProvider::read_attribute(test_env) == InfraProvider::K8s { - args.ipv6_config_type = Some("Fixed".to_string()); - args.fixed_address = Some(format!("{}/64", node.node_config.public_api.ip())); - args.fixed_gateway = Some("fe80::ecee:eeff:feee:eeee".to_string()); + config.ipv6_config_type = Some("Fixed".to_string()); + config.fixed_address = Some(format!("{}/64", node.node_config.public_api.ip())); + config.fixed_gateway = Some("fe80::ecee:eeff:feee:eeee".to_string()); } // If we have a root subnet, specify the correct NNS url. @@ -461,7 +459,7 @@ fn create_config_disk_image( .nodes() .next() { - args.nns_urls = Some(vec![format!("http://[{}]:8080", node.get_ip_addr())]); + config.nns_urls = Some(vec![format!("http://[{}]:8080", node.get_ip_addr())]); } if let Some(malicious_behavior) = malicious_behavior { @@ -469,7 +467,7 @@ fn create_config_disk_image( test_env.logger(), "Node with id={} has malicious behavior={:?}", node.node_id, malicious_behavior ); - args.malicious_behavior = Some(serde_json::to_string(&malicious_behavior)?); + config.malicious_behavior = Some(serde_json::to_string(&malicious_behavior)?); } if let Some(query_stats_epoch_length) = query_stats_epoch_length { @@ -479,7 +477,7 @@ fn create_config_disk_image( node.node_id, query_stats_epoch_length ); - args.query_stats_epoch_length = Some(query_stats_epoch_length); + config.query_stats_epoch_length = Some(query_stats_epoch_length); } if let Some(ipv4_config) = ipv4_config { @@ -487,9 +485,9 @@ fn create_config_disk_image( test_env.logger(), "Node with id={} is IPv4-enabled: {:?}", node.node_id, ipv4_config ); - args.ipv4_address = Some(ipv4_config.ip_addr().to_string()); - args.ipv4_gateway = Some(ipv4_config.gateway_ip_addr().to_string()); - args.ipv4_prefix_length = Some(ipv4_config.prefix_length().try_into().unwrap()); + config.ipv4_address = Some(ipv4_config.ip_addr().to_string()); + config.ipv4_gateway = Some(ipv4_config.gateway_ip_addr().to_string()); + config.ipv4_prefix_length = Some(ipv4_config.prefix_length().try_into().unwrap()); } if let Some(domain) = domain { @@ -497,7 +495,7 @@ fn create_config_disk_image( test_env.logger(), "Node with id={} has domain_name {}", node.node_id, domain, ); - args.ipv4_domain = Some(domain); + config.ipv4_domain = Some(domain); } let elasticsearch_hosts: Vec = get_elasticsearch_hosts()?; @@ -506,26 +504,27 @@ fn create_config_disk_image( "ElasticSearch hosts are {:?}", elasticsearch_hosts ); if !elasticsearch_hosts.is_empty() { - args.elasticsearch_hosts = Some(elasticsearch_hosts.join(" ")); + config.elasticsearch_hosts = Some(elasticsearch_hosts.join(" ")); } // --bitcoind_addr indicates the local bitcoin node that the bitcoin adapter should be connected to in the system test environment. if let Ok(bitcoin_addr) = test_env.read_json_object::(BITCOIND_ADDR_PATH) { - args.bitcoind_addr = Some(bitcoin_addr); + config.bitcoind_addr = Some(bitcoin_addr); } // --jaeger_addr indicates the local Jaeger node that the nodes should be connected to in the system test environment. if let Ok(jaeger_addr) = test_env.read_json_object::(JAEGER_ADDR_PATH) { - args.jaeger_addr = Some(jaeger_addr); + config.jaeger_addr = Some(jaeger_addr); } // --socks_proxy indicates that a socks proxy is available to the system test environment. if let Ok(socks_proxy) = test_env.read_json_object::(SOCKS_PROXY_PATH) { - args.socks_proxy = Some(socks_proxy); + config.socks_proxy = Some(socks_proxy); } // populate guestos_config_json_path with serialized guestos config object - generate_testnet_config(args)?; + let guestos_config_json_path = tempdir().unwrap().as_ref().join("guestos_config.json"); + generate_testnet_config(config, guestos_config_json_path)?; let img_path = PathBuf::from(&node.node_path).join(CONF_IMG_FNAME); let script_path = From 0c0051beb8b0f7b27a33ea84b0c433545b5b2436 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 25 Oct 2024 22:11:25 +0000 Subject: [PATCH 131/241] Add hostname field to config types --- .../components/early-boot/setup-hostname/setup-hostname.sh | 6 +++++- rs/ic_os/config/src/generate_testnet_config.rs | 3 +++ rs/ic_os/config/src/main.rs | 3 +++ rs/ic_os/config/src/types.rs | 2 ++ rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs | 3 ++- rs/tests/driver/src/driver/bootstrap.rs | 5 ++++- 6 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh index e39406d4887..f105b3bfa64 100755 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh @@ -51,10 +51,14 @@ function validate_arguments() { function read_config_variables() { mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') mgmt_mac=${mgmt_mac//:/} # Remove colons from mgmt_mac + config_hostname=$(get_config_value '.guestos_settings.guestos_dev_settings.hostname') } function construct_hostname() { - if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then + if [ "${config_hostname}" != "" ] && [ "${config_hostname}" != "null" ]; then + HOSTNAME=${config_hostname} + write_log "Using manually configured hostname: ${HOSTNAME}" + elif [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then HOSTNAME=$(echo ${TYPE}-${mgmt_mac}-$(cat ${FILE})) write_log "Using hostname: ${HOSTNAME}" write_metric "setup_hostname" \ diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 6ba292e3245..333f132f3b4 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -43,6 +43,7 @@ pub struct GenerateTestnetConfigArgs { pub bitcoind_addr: Option, pub jaeger_addr: Option, pub socks_proxy: Option, + pub hostname: Option, } /// Generates a writes a serialized GuestOSConfig to guestos_config_json_path @@ -80,6 +81,7 @@ pub fn generate_testnet_config( bitcoind_addr, jaeger_addr, socks_proxy, + hostname, } = config; // Construct the NetworkSettings @@ -214,6 +216,7 @@ pub fn generate_testnet_config( bitcoind_addr, jaeger_addr, socks_proxy, + hostname, }; // Construct GuestOSSettings diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 722b310d93d..e6fe24bf935 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -124,6 +124,8 @@ pub struct GenerateTestnetConfigClapArgs { pub jaeger_addr: Option, #[arg(long)] pub socks_proxy: Option, + #[arg(long)] + pub hostname: Option, // Output path #[arg(long)] @@ -370,6 +372,7 @@ pub fn main() -> Result<()> { bitcoind_addr: clap_args.bitcoind_addr, jaeger_addr: clap_args.jaeger_addr, socks_proxy: clap_args.socks_proxy, + hostname: clap_args.hostname, }; generate_testnet_config(args, clap_args.guestos_config_json_path) diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 7fc45ef39ad..74769f0303e 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -71,6 +71,8 @@ pub struct GuestOSDevSettings { pub bitcoind_addr: Option, pub jaeger_addr: Option, pub socks_proxy: Option, + // An optional hostname to override the deterministically generated hostname + pub hostname: Option, } /// Configures the usage of the backup spool directory. diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index 1b0f4e64fc1..c4b11e638ba 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -232,11 +232,12 @@ fn main() { bitcoind_addr: None, jaeger_addr: None, socks_proxy: None, + hostname: None, }; // populate guestos_config_json_path with serialized guestos config object let guestos_config_json_path = tempdir.as_ref().join("guestos_config.json"); - let _ = generate_testnet_config(config, guestos_config_json_path); + let _ = generate_testnet_config(config, guestos_config_json_path.clone()); // Build config image let filename = "config.tar.gz"; diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index f55f5f2c562..7a3e88c685b 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -442,6 +442,7 @@ fn create_config_disk_image( bitcoind_addr: None, jaeger_addr: None, socks_proxy: None, + hostname: None, }; // We've seen k8s nodes fail to pick up RA correctly, so we specify their @@ -522,9 +523,11 @@ fn create_config_disk_image( config.socks_proxy = Some(socks_proxy); } + config.hostname = Some(node.node_id.to_string()); + // populate guestos_config_json_path with serialized guestos config object let guestos_config_json_path = tempdir().unwrap().as_ref().join("guestos_config.json"); - generate_testnet_config(config, guestos_config_json_path)?; + generate_testnet_config(config, guestos_config_json_path.clone())?; let img_path = PathBuf::from(&node.node_path).join(CONF_IMG_FNAME); let script_path = From b22fef7224522d012adc8e0efb9aafc6b7d6fe5f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 25 Oct 2024 23:06:53 +0000 Subject: [PATCH 132/241] rename inject config values and make bool --- rs/ic_os/config/src/generate_testnet_config.rs | 18 +++++++++--------- rs/ic_os/config/src/lib.rs | 6 +++--- rs/ic_os/config/src/main.rs | 12 ++++++------ rs/ic_os/config/src/types.rs | 6 +++--- .../launch-single-vm/src/main.rs | 6 +++--- rs/tests/driver/src/driver/bootstrap.rs | 6 +++--- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 333f132f3b4..5f9de4c5044 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -31,9 +31,9 @@ pub struct GenerateTestnetConfigArgs { pub ssh_authorized_keys_path: Option, // GuestOSSettings arguments - pub ic_crypto_path: Option, - pub ic_state_path: Option, - pub ic_registry_local_store_path: Option, + pub inject_ic_crypto: Option, + pub inject_ic_state: Option, + pub inject_ic_registry_local_store: Option, // GuestOSDevSettings arguments pub backup_retention_time_seconds: Option, @@ -71,9 +71,9 @@ pub fn generate_testnet_config( nns_urls, node_operator_private_key_path, ssh_authorized_keys_path, - ic_crypto_path, - ic_state_path, - ic_registry_local_store_path, + inject_ic_crypto, + inject_ic_state, + inject_ic_registry_local_store, backup_retention_time_seconds, backup_purging_interval_seconds, malicious_behavior, @@ -221,9 +221,9 @@ pub fn generate_testnet_config( // Construct GuestOSSettings let guestos_settings = GuestOSSettings { - ic_crypto_path, - ic_state_path, - ic_registry_local_store_path, + inject_ic_crypto: inject_ic_crypto.unwrap_or(false), + inject_ic_state: inject_ic_state.unwrap_or(false), + inject_ic_registry_local_store: inject_ic_registry_local_store.unwrap_or(false), guestos_dev_settings, }; diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 5b3f3c8e67d..bb7db390eaf 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -91,9 +91,9 @@ mod tests { verbose: false, }; let guestos_settings = GuestOSSettings { - ic_crypto_path: None, - ic_state_path: None, - ic_registry_local_store_path: None, + inject_ic_crypto: false, + inject_ic_state: false, + inject_ic_registry_local_store: false, guestos_dev_settings: GuestOSDevSettings::default(), }; diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index e6fe24bf935..3df1979e916 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -103,11 +103,11 @@ pub struct GenerateTestnetConfigClapArgs { // GuestOSSettings arguments #[arg(long)] - pub ic_crypto_path: Option, + pub inject_ic_crypto: Option, #[arg(long)] - pub ic_state_path: Option, + pub inject_ic_state: Option, #[arg(long)] - pub ic_registry_local_store_path: Option, + pub inject_ic_registry_local_store: Option, // GuestOSDevSettings arguments #[arg(long)] @@ -362,9 +362,9 @@ pub fn main() -> Result<()> { nns_urls: clap_args.nns_urls, node_operator_private_key_path: clap_args.node_operator_private_key_path, ssh_authorized_keys_path: clap_args.ssh_authorized_keys_path, - ic_crypto_path: clap_args.ic_crypto_path, - ic_state_path: clap_args.ic_state_path, - ic_registry_local_store_path: clap_args.ic_registry_local_store_path, + inject_ic_crypto: clap_args.inject_ic_crypto, + inject_ic_state: clap_args.inject_ic_state, + inject_ic_registry_local_store: clap_args.inject_ic_registry_local_store, backup_retention_time_seconds: clap_args.backup_retention_time_seconds, backup_purging_interval_seconds: clap_args.backup_purging_interval_seconds, malicious_behavior: clap_args.malicious_behavior, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 74769f0303e..b302caaa5a2 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -52,13 +52,13 @@ pub struct GuestOSSettings { /// Must be a directory with contents matching the internal representation of the ic_crypto directory. /// When given, this provides the private keys of the node. /// If not given, the node will generate its own private/public key pair. - pub ic_crypto_path: Option, - pub ic_state_path: Option, + pub inject_ic_crypto: bool, + pub inject_ic_state: bool, /// Initial registry state. /// Must be a directory with contents matching the internal representation of the ic_registry_local_store. /// When given, this provides the initial state of the registry. /// If not given, the node will fetch (initial) registry state from the NNS. - pub ic_registry_local_store_path: Option, + pub inject_ic_registry_local_store: bool, pub guestos_dev_settings: GuestOSDevSettings, } diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index c4b11e638ba..87192fbe589 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -222,9 +222,9 @@ fn main() { nns_urls: Some(vec![format!("http://[{}]", ipv6_addr)]), node_operator_private_key_path: None, ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), - ic_crypto_path: None, - ic_state_path: None, - ic_registry_local_store_path: None, + inject_ic_crypto: Some(false), + inject_ic_state: Some(false), + inject_ic_registry_local_store: Some(false), backup_retention_time_seconds: None, backup_purging_interval_seconds: None, malicious_behavior: None, diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 7a3e88c685b..508fd3bf467 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -432,9 +432,9 @@ fn create_config_disk_image( nns_urls: None, node_operator_private_key_path: None, ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), - ic_crypto_path: None, - ic_state_path: None, - ic_registry_local_store_path: None, + inject_ic_crypto: Some(false), + inject_ic_state: Some(false), + inject_ic_registry_local_store: Some(false), backup_retention_time_seconds: None, backup_purging_interval_seconds: None, malicious_behavior: None, From 2294217dda2d3d751bd7a422cf86ad8d6e28e312 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 25 Oct 2024 23:10:26 +0000 Subject: [PATCH 133/241] Update config.sh config references --- ic-os/components/misc/config/config.sh | 2 +- ic-os/components/misc/config/setupos/config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/misc/config/config.sh b/ic-os/components/misc/config/config.sh index 7cd800317d8..5cc0f222b2c 100644 --- a/ic-os/components/misc/config/config.sh +++ b/ic-os/components/misc/config/config.sh @@ -4,7 +4,7 @@ # Retrieves a value from the config.json file using a JSON path. # Arguments: -# $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') +# $1 - JSON path to the desired value (e.g., '.icos_settings.nns_urls') # Note: If the key is not found, this function will return null. function get_config_value() { local CONFIG_FILE="/boot/config/config.json" diff --git a/ic-os/components/misc/config/setupos/config.sh b/ic-os/components/misc/config/setupos/config.sh index 57198454f99..a16d9fcceba 100644 --- a/ic-os/components/misc/config/setupos/config.sh +++ b/ic-os/components/misc/config/setupos/config.sh @@ -4,7 +4,7 @@ # Retrieves a value from the config.json file using a JSON path. # Arguments: -# $1 - JSON path to the desired value (e.g., '.icos_settings.node_operator_private_key_path') +# $1 - JSON path to the desired value (e.g., '.icos_settings.nns_urls') # Note: If the key is not found, this function will return null. function get_config_value() { local CONFIG_FILE="/var/ic/config/config.json" From a2d6ed0021bfef8dd1901ee2652b0406ea06773d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 28 Oct 2024 18:09:44 +0000 Subject: [PATCH 134/241] Rename nns_public_key_exists and node_operator_private_key_exists and update to bool --- .../dev-generate-guestos-config.sh | 12 ++-- .../generate-guestos-config.sh | 12 ++-- .../setupos-scripts/check-hardware.sh | 4 +- .../setupos-scripts/setup-hostos-config.sh | 56 +++++++++++-------- ic-os/components/setupos-scripts/setupos.sh | 4 +- .../config/src/generate_testnet_config.rs | 17 +++--- rs/ic_os/config/src/lib.rs | 8 +-- rs/ic_os/config/src/main.rs | 31 ++++------ rs/ic_os/config/src/types.rs | 7 +-- .../launch-single-vm/src/main.rs | 4 +- rs/tests/driver/src/driver/bootstrap.rs | 4 +- 11 files changed, 78 insertions(+), 81 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index d0445d76da3..7be78e0dae6 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -56,9 +56,9 @@ OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') - nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') + nns_public_key_exists=$(get_config_value '.icos_settings.nns_public_key_exists') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') - node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') + node_operator_private_key_exists=$(get_config_value '.icos_settings.node_operator_private_key_exists') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') ssh_authorized_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') @@ -70,9 +70,11 @@ function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--guestos_config "/boot/config/config-guestos.json") - cmd+=(--nns_public_key "$nns_public_key") - if [ -f "$node_operator_private_key" ]; then - cmd+=(--node_operator_private_key "$node_operator_private_key") + if [[ "${nns_public_key_exists,,}" == "true" ]]; then + cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") + fi + if [ -f "$node_operator_private_key_exists" ]; then + cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi cmd+=(--accounts_ssh_authorized_keys "$ssh_authorized_keys") diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 3cf5292dfff..bdd682b22fd 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -56,9 +56,9 @@ OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') - nns_public_key=$(get_config_value '.icos_settings.nns_public_key_path') + nns_public_key_exists=$(get_config_value '.icos_settings.nns_public_key_exists') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') - node_operator_private_key=$(get_config_value '.icos_settings.node_operator_private_key_path') + node_operator_private_key_exists=$(get_config_value '.icos_settings.node_operator_private_key_exists') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') } @@ -69,9 +69,11 @@ function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--guestos_config "/boot/config/config-guestos.json") - cmd+=(--nns_public_key "$nns_public_key") - if [ -f "$node_operator_private_key" ]; then - cmd+=(--node_operator_private_key "$node_operator_private_key") + if [[ "${nns_public_key_exists,,}" == "true" ]]; then + cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") + fi + if [ -f "$node_operator_private_key_exists" ]; then + cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi # Run the above command diff --git a/ic-os/components/setupos-scripts/check-hardware.sh b/ic-os/components/setupos-scripts/check-hardware.sh index 96e7eb2ba91..3e77112fe75 100644 --- a/ic-os/components/setupos-scripts/check-hardware.sh +++ b/ic-os/components/setupos-scripts/check-hardware.sh @@ -249,9 +249,7 @@ function verify_disks() { function verify_deployment_path() { echo "* Verifying deployment path..." - local node_operator_key_path=$(get_config_value '.icos_settings.node_operator_private_key_path') - - if [[ ${GENERATION} == 2 ]] && [[ ! -f "${node_operator_key_path}" ]]; then + if [[ ${GENERATION} == 2 ]] && [[ ! -f "/boot/config/node_operator_private_key.pem" ]]; then echo -e "\n\n\n\n\n\n" echo -e "\033[1;31mWARNING: Gen2 hardware detected but no Node Operator Private Key found.\033[0m" echo -e "\033[1;31mGen2 hardware should be deployed using the Gen2 Node Deployment method.\033[0m" diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 5ec99deb3a8..483f671f4ae 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -35,20 +35,31 @@ function copy_config_files() { fi echo "* Copying node operator private key..." - node_operator_private_key_path=$(get_config_value '.icos_settings.node_operator_private_key_path') - if [ "${node_operator_private_key_path}" != "null" ] && [ -f "${node_operator_private_key_path}" ]; then - cp "${node_operator_private_key_path}" /media/ - log_and_halt_installation_on_error "${?}" "Unable to copy node operator private key to hostOS config partition." - elif [ "${node_operator_private_key_path}" = "null" ]; then - echo >&2 "Warning: Node operator private key path is not configured." + node_operator_private_key_exists=$(get_config_value '.icos_settings.node_operator_private_key_exists') + if [[ "${node_operator_private_key_exists,,}" == "true" ]]; then + if [ -f "${CONFIG_DIR}/node_operator_private_key.pem" ]; then + cp "${CONFIG_DIR}/node_operator_private_key.pem" /media/ + log_and_halt_installation_on_error "${?}" "Unable to copy node operator private key to hostOS config partition." + else + log_and_halt_installation_on_error "1" "node_operator_private_key_exists set to true but not found" + fi else echo >&2 "Warning: node_operator_private_key.pem does not exist, requiring HSM." + insert_hsm fi echo "* Copying NNS public key to hostOS config partition..." - nns_public_key_path=$(get_config_value '.icos_settings.nns_public_key_path') - cp "${nns_public_key_path}" /media/ - log_and_halt_installation_on_error "${?}" "Unable to copy NNS public key to hostOS config partition." + nns_public_key_exists=$(get_config_value '.icos_settings.nns_public_key_exists') + if [[ "${nns_public_key_exists,,}" == "true" ]]; then + if [ -f "/data/nns_public_key.pem" ]; then + cp /data/nns_public_key.pem /media/ + log_and_halt_installation_on_error "${?}" "Unable to copy NNS public key to hostOS config partition." + else + log_and_halt_installation_on_error "1" "nns_public_key_exists set to true but not found." + fi + else + log_and_halt_installation_on_error "1" "nns_public_key_exists must be set to true." + fi echo "* Converting 'config.json' to hostOS config file 'config-hostos.json'..." /opt/ic/bin/config generate-hostos-config @@ -63,20 +74,18 @@ function copy_config_files() { fi } -function insert_hsm_if_necessary() { - if [ ! -f "${CONFIG_DIR}/node_operator_private_key.pem" ]; then - retry=0 - while [ -z "$(lsusb | grep -E 'Nitro|Clay')" ]; do - let retry=retry+1 - if [ ${retry} -ge 3600 ]; then - log_and_halt_installation_on_error "1" "Nitrokey HSM USB device could not be detected, giving up." - break - else - echo "* Please insert Nitrokey HSM USB device..." - sleep 3 - fi - done - fi +function insert_hsm() { + retry=0 + while [ -z "$(lsusb | grep -E 'Nitro|Clay')" ]; do + let retry=retry+1 + if [ ${retry} -ge 3600 ]; then + log_and_halt_installation_on_error "1" "Nitrokey HSM USB device could not be detected, giving up." + break + else + echo "* Please insert Nitrokey HSM USB device..." + sleep 3 + fi + done } function unmount_config_partition() { @@ -97,7 +106,6 @@ main() { log_start "$(basename $0)" mount_config_partition copy_config_files - insert_hsm_if_necessary unmount_config_partition log_end "$(basename $0)" } diff --git a/ic-os/components/setupos-scripts/setupos.sh b/ic-os/components/setupos-scripts/setupos.sh index 1c4d5645e58..82e6b096587 100755 --- a/ic-os/components/setupos-scripts/setupos.sh +++ b/ic-os/components/setupos-scripts/setupos.sh @@ -40,8 +40,8 @@ main() { start_setupos /opt/ic/bin/check-setupos-age.sh /opt/ic/bin/check-config.sh - /opt/ic/bin/check-hardware.sh - /opt/ic/bin/check-network.sh + #/opt/ic/bin/check-hardware.sh + #/opt/ic/bin/check-network.sh /opt/ic/bin/setup-disk.sh /opt/ic/bin/install-hostos.sh /opt/ic/bin/install-guestos.sh diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 5f9de4c5044..82b69873c3d 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -25,9 +25,9 @@ pub struct GenerateTestnetConfigArgs { pub deployment_environment: Option, pub elasticsearch_hosts: Option, pub elasticsearch_tags: Option, - pub nns_public_key_path: Option, + pub nns_public_key_exists: Option, pub nns_urls: Option>, - pub node_operator_private_key_path: Option, + pub node_operator_private_key_exists: Option, pub ssh_authorized_keys_path: Option, // GuestOSSettings arguments @@ -67,9 +67,9 @@ pub fn generate_testnet_config( deployment_environment, elasticsearch_hosts, elasticsearch_tags, - nns_public_key_path, + nns_public_key_exists, nns_urls, - node_operator_private_key_path, + node_operator_private_key_exists, ssh_authorized_keys_path, inject_ic_crypto, inject_ic_state, @@ -170,8 +170,7 @@ pub fn generate_testnet_config( elasticsearch_tags, }; - let nns_public_key_path = - nns_public_key_path.unwrap_or_else(|| PathBuf::from("/boot/config/nns_public_key.pem")); + let nns_public_key_exists = nns_public_key_exists.unwrap_or(true); let nns_urls = match nns_urls { Some(urls) => urls @@ -181,13 +180,15 @@ pub fn generate_testnet_config( None => vec![Url::parse("https://wiki.internetcomputer.org")?], }; + let node_operator_private_key_exists = node_operator_private_key_exists.unwrap_or(false); + let icos_settings = ICOSSettings { mgmt_mac, deployment_environment, logging, - nns_public_key_path, + nns_public_key_exists, nns_urls, - node_operator_private_key_path, + node_operator_private_key_exists, ssh_authorized_keys_path, icos_dev_settings: ICOSDevSettings::default(), }; diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index bb7db390eaf..646394d38db 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -12,10 +12,7 @@ use std::path::Path; pub static DEFAULT_SETUPOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/config.json"; pub static DEFAULT_SETUPOS_CONFIG_INI_FILE_PATH: &str = "/config/config.ini"; pub static DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH: &str = "/data/deployment.json"; -pub static DEFAULT_SETUPOS_NNS_PUBLIC_KEY_PATH: &str = "/data/nns_public_key.pem"; pub static DEFAULT_SETUPOS_SSH_AUTHORIZED_KEYS_PATH: &str = "/config/ssh_authorized_keys"; -pub static DEFAULT_SETUPOS_NODE_OPERATOR_PRIVATE_KEY_PATH: &str = - "/config/node_operator_private_key.pem"; pub static DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/config-hostos.json"; @@ -49,7 +46,6 @@ pub fn deserialize_config Deserialize<'de>, P: AsRef>(file_pat mod tests { use super::*; use mac_address::mac_address::FormattedMacAddress; - use std::path::PathBuf; use types::*; #[test] @@ -78,9 +74,9 @@ mod tests { mgmt_mac: FormattedMacAddress::try_from("ec:2a:72:31:a2:0c")?, deployment_environment: "Mainnet".to_string(), logging, - nns_public_key_path: PathBuf::from("/path/to/key"), + nns_public_key_exists: true, nns_urls: vec!["http://localhost".parse().unwrap()], - node_operator_private_key_path: None, + node_operator_private_key_exists: true, ssh_authorized_keys_path: None, icos_dev_settings, }; diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 3df1979e916..ed8275e25f4 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -21,14 +21,14 @@ pub enum Commands { #[arg(long, default_value = config::DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH, value_name = "deployment.json")] deployment_json_path: PathBuf, - #[arg(long, default_value = config::DEFAULT_SETUPOS_NNS_PUBLIC_KEY_PATH, value_name = "nns_public_key.pem")] - nns_public_key_path: PathBuf, + #[arg(long, default_value_t = true)] + nns_public_key_exists: bool, #[arg(long, default_value = config::DEFAULT_SETUPOS_SSH_AUTHORIZED_KEYS_PATH, value_name = "ssh_authorized_keys")] ssh_authorized_keys_path: PathBuf, - #[arg(long, default_value = config::DEFAULT_SETUPOS_NODE_OPERATOR_PRIVATE_KEY_PATH, value_name = "node_operator_private_key.pem")] - node_operator_private_key_path: PathBuf, + #[arg(long, default_value_t = true)] + node_operator_private_key_exists: bool, #[arg(long, default_value = config::DEFAULT_SETUPOS_CONFIG_OBJECT_PATH, value_name = "config.json")] setupos_config_json_path: PathBuf, @@ -93,11 +93,11 @@ pub struct GenerateTestnetConfigClapArgs { #[arg(long)] pub elasticsearch_tags: Option, #[arg(long)] - pub nns_public_key_path: Option, + pub nns_public_key_exists: Option, #[arg(long)] pub nns_urls: Option>, #[arg(long)] - pub node_operator_private_key_path: Option, + pub node_operator_private_key_exists: Option, #[arg(long)] pub ssh_authorized_keys_path: Option, @@ -139,9 +139,9 @@ pub fn main() -> Result<()> { Some(Commands::CreateSetuposConfig { config_ini_path, deployment_json_path, - nns_public_key_path, + nns_public_key_exists, ssh_authorized_keys_path, - node_operator_private_key_path, + node_operator_private_key_exists, setupos_config_json_path, }) => { // get config.ini settings @@ -208,11 +208,9 @@ pub fn main() -> Result<()> { mgmt_mac, deployment_environment: deployment_json_settings.deployment.name, logging, - nns_public_key_path: nns_public_key_path.to_path_buf(), + nns_public_key_exists, nns_urls: deployment_json_settings.nns.url.clone(), - node_operator_private_key_path: node_operator_private_key_path - .exists() - .then_some(node_operator_private_key_path), + node_operator_private_key_exists, ssh_authorized_keys_path: ssh_authorized_keys_path .exists() .then_some(ssh_authorized_keys_path), @@ -267,11 +265,6 @@ pub fn main() -> Result<()> { if let Some(ref mut path) = hostos_icos_settings.ssh_authorized_keys_path { *path = hostos_config_path.join("ssh_authorized_keys"); } - if let Some(ref mut path) = hostos_icos_settings.node_operator_private_key_path { - *path = hostos_config_path.join("node_operator_private_key.pem"); - } - hostos_icos_settings.nns_public_key_path = - hostos_config_path.join("nns_public_key.pem"); let hostos_config = HostOSConfig { network_settings: setupos_config.network_settings, @@ -358,9 +351,9 @@ pub fn main() -> Result<()> { deployment_environment: clap_args.deployment_environment, elasticsearch_hosts: clap_args.elasticsearch_hosts, elasticsearch_tags: clap_args.elasticsearch_tags, - nns_public_key_path: clap_args.nns_public_key_path, + nns_public_key_exists: clap_args.nns_public_key_exists, nns_urls: clap_args.nns_urls, - node_operator_private_key_path: clap_args.node_operator_private_key_path, + node_operator_private_key_exists: clap_args.node_operator_private_key_exists, ssh_authorized_keys_path: clap_args.ssh_authorized_keys_path, inject_ic_crypto: clap_args.inject_ic_crypto, inject_ic_state: clap_args.inject_ic_state, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index b302caaa5a2..f9c4c8114eb 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -92,13 +92,10 @@ pub struct ICOSSettings { /// "mainnet" or "testnet" pub deployment_environment: String, pub logging: Logging, - /// This file must be a text file containing the public key of the NNS to be used. - pub nns_public_key_path: PathBuf, + pub nns_public_key_exists: bool, /// The URL (HTTP) of the NNS node(s). pub nns_urls: Vec, - /// This file contains the Node Operator private key, - /// which is registered with the NNS and used to sign the IC join request. - pub node_operator_private_key_path: Option, + pub node_operator_private_key_exists: bool, /// This directory contains individual files named `admin`, `backup`, `readonly`. /// The contents of these files serve as `authorized_keys` for their respective role account. /// This means that, for example, `accounts_ssh_authorized_keys/admin` diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index 87192fbe589..eef3a6d8521 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -218,9 +218,9 @@ fn main() { deployment_environment: Some("testnet".to_string()), elasticsearch_hosts: None, elasticsearch_tags: None, - nns_public_key_path: None, + nns_public_key_exists: Some(true), nns_urls: Some(vec![format!("http://[{}]", ipv6_addr)]), - node_operator_private_key_path: None, + node_operator_private_key_exists: Some(true), ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), inject_ic_crypto: Some(false), inject_ic_state: Some(false), diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 508fd3bf467..bcb121c3926 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -428,9 +428,9 @@ fn create_config_disk_image( deployment_environment: Some("testnet".to_string()), elasticsearch_hosts: None, elasticsearch_tags: Some(format!("system_test {}", group_name)), - nns_public_key_path: None, + nns_public_key_exists: Some(true), nns_urls: None, - node_operator_private_key_path: None, + node_operator_private_key_exists: Some(true), ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), inject_ic_crypto: Some(false), inject_ic_state: Some(false), From 3ccd4bf62069959aed7a588664c13f3721402975 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 28 Oct 2024 18:24:17 +0000 Subject: [PATCH 135/241] Update use_ssh_authorized_keys to bool --- .../dev-generate-guestos-config.sh | 6 ++-- .../setupos-scripts/setup-hostos-config.sh | 8 ++--- .../setup-ssh-account-keys.sh | 14 +++----- .../config/src/generate_testnet_config.rs | 8 +++-- rs/ic_os/config/src/lib.rs | 2 +- rs/ic_os/config/src/main.rs | 32 +++++-------------- rs/ic_os/config/src/types.rs | 6 ++-- .../launch-single-vm/src/main.rs | 2 +- rs/tests/driver/src/driver/bootstrap.rs | 2 +- 9 files changed, 31 insertions(+), 49 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 7be78e0dae6..08c377c805a 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -61,7 +61,7 @@ function read_config_variables() { node_operator_private_key_exists=$(get_config_value '.icos_settings.node_operator_private_key_exists') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') - ssh_authorized_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') + use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') } function assemble_config_media() { @@ -76,7 +76,9 @@ function assemble_config_media() { if [ -f "$node_operator_private_key_exists" ]; then cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi - cmd+=(--accounts_ssh_authorized_keys "$ssh_authorized_keys") + if [[ "${ssh_authorized_keys,,}" == "true" ]]; then + cmd+=(--accounts_ssh_authorized_keys "/boot/config/ssh_authorized_keys") + fi # Run the above command "${cmd[@]}" diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 483f671f4ae..66f130d075d 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -22,16 +22,16 @@ function mount_config_partition() { function copy_config_files() { echo "* Copying SSH authorized keys..." - ssh_authorized_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') - if [ -n "${ssh_authorized_keys}" ] && [ "${ssh_authorized_keys}" != "null" ]; then + use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') + if [[ "${ssh_authorized_keys,,}" == "true" ]]; then if [ -d "${ssh_authorized_keys}" ]; then cp -a "${ssh_authorized_keys}" /media/ log_and_halt_installation_on_error "${?}" "Unable to copy SSH authorized keys to hostOS config partition." else - log_and_halt_installation_on_error "1" "Directory '${ssh_authorized_keys}' does not exist." + echo >&2 "Warning: SSH authorized keys are not configured." fi else - echo >&2 "Warning: SSH authorized keys path is not configured." + echo >&2 "SSH keys not in use." fi echo "* Copying node operator private key..." diff --git a/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh b/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh index 18731902bfa..d1106e2373d 100755 --- a/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh +++ b/ic-os/components/ssh/setup-ssh-account-keys/setup-ssh-account-keys.sh @@ -2,12 +2,6 @@ set -e -source /opt/ic/bin/config.sh - -read_config_variables() { - authorized_ssh_keys=$(get_config_value '.icos_settings.ssh_authorized_keys_path') -} - copy_ssh_keys() { local SOURCE_FILE="$1" local DEST_FILE="$2" @@ -17,8 +11,6 @@ copy_ssh_keys() { fi } -read_config_variables - for ACCOUNT in backup readonly admin; do HOMEDIR=$(getent passwd "${ACCOUNT}" | cut -d: -f6) GROUP=$(id -ng "${ACCOUNT}") @@ -26,10 +18,12 @@ for ACCOUNT in backup readonly admin; do mkdir -p "${HOMEDIR}/.ssh" chmod 700 "${HOMEDIR}" "${HOMEDIR}/.ssh" - AUTHORIZED_SSH_KEYS="${authorized_ssh_keys}/${ACCOUNT}" + GUESTOS_AUTHORIZED_SSH_KEYS="/boot/config/accounts_ssh_authorized_keys/${ACCOUNT}" + HOSTOS_AUTHORIZED_SSH_KEYS="/boot/config/ssh_authorized_keys/${ACCOUNT}" AUTHORIZED_KEYS_FILE="${HOMEDIR}/.ssh/authorized_keys" - copy_ssh_keys "${AUTHORIZED_SSH_KEYS}" "${AUTHORIZED_KEYS_FILE}" + copy_ssh_keys "${GUESTOS_AUTHORIZED_SSH_KEYS}" "${AUTHORIZED_KEYS_FILE}" + copy_ssh_keys "${HOSTOS_AUTHORIZED_SSH_KEYS}" "${AUTHORIZED_KEYS_FILE}" chown -R "${ACCOUNT}:${GROUP}" "${HOMEDIR}" restorecon -r "${HOMEDIR}" diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 82b69873c3d..2d23d8d84fb 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -28,7 +28,7 @@ pub struct GenerateTestnetConfigArgs { pub nns_public_key_exists: Option, pub nns_urls: Option>, pub node_operator_private_key_exists: Option, - pub ssh_authorized_keys_path: Option, + pub use_ssh_authorized_keys: Option, // GuestOSSettings arguments pub inject_ic_crypto: Option, @@ -70,7 +70,7 @@ pub fn generate_testnet_config( nns_public_key_exists, nns_urls, node_operator_private_key_exists, - ssh_authorized_keys_path, + use_ssh_authorized_keys, inject_ic_crypto, inject_ic_state, inject_ic_registry_local_store, @@ -182,6 +182,8 @@ pub fn generate_testnet_config( let node_operator_private_key_exists = node_operator_private_key_exists.unwrap_or(false); + let use_ssh_authorized_keys = use_ssh_authorized_keys.unwrap_or(true); + let icos_settings = ICOSSettings { mgmt_mac, deployment_environment, @@ -189,7 +191,7 @@ pub fn generate_testnet_config( nns_public_key_exists, nns_urls, node_operator_private_key_exists, - ssh_authorized_keys_path, + use_ssh_authorized_keys, icos_dev_settings: ICOSDevSettings::default(), }; diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 646394d38db..e4a519427b1 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -77,7 +77,7 @@ mod tests { nns_public_key_exists: true, nns_urls: vec!["http://localhost".parse().unwrap()], node_operator_private_key_exists: true, - ssh_authorized_keys_path: None, + use_ssh_authorized_keys: false, icos_dev_settings, }; let setupos_settings = SetupOSSettings; diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index ed8275e25f4..ad772a39291 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -24,8 +24,8 @@ pub enum Commands { #[arg(long, default_value_t = true)] nns_public_key_exists: bool, - #[arg(long, default_value = config::DEFAULT_SETUPOS_SSH_AUTHORIZED_KEYS_PATH, value_name = "ssh_authorized_keys")] - ssh_authorized_keys_path: PathBuf, + #[arg(long, default_value_t = false)] + use_ssh_authorized_keys: bool, #[arg(long, default_value_t = true)] node_operator_private_key_exists: bool, @@ -99,7 +99,7 @@ pub struct GenerateTestnetConfigClapArgs { #[arg(long)] pub node_operator_private_key_exists: Option, #[arg(long)] - pub ssh_authorized_keys_path: Option, + pub use_ssh_authorized_keys: Option, // GuestOSSettings arguments #[arg(long)] @@ -140,7 +140,7 @@ pub fn main() -> Result<()> { config_ini_path, deployment_json_path, nns_public_key_exists, - ssh_authorized_keys_path, + use_ssh_authorized_keys, node_operator_private_key_exists, setupos_config_json_path, }) => { @@ -211,9 +211,7 @@ pub fn main() -> Result<()> { nns_public_key_exists, nns_urls: deployment_json_settings.nns.url.clone(), node_operator_private_key_exists, - ssh_authorized_keys_path: ssh_authorized_keys_path - .exists() - .then_some(ssh_authorized_keys_path), + use_ssh_authorized_keys, icos_dev_settings: ICOSDevSettings::default(), }; @@ -259,16 +257,9 @@ pub fn main() -> Result<()> { let setupos_config: SetupOSConfig = serde_json::from_reader(File::open(setupos_config_json_path)?)?; - // update select file paths for HostOS - let mut hostos_icos_settings = setupos_config.icos_settings; - let hostos_config_path = Path::new("/boot/config"); - if let Some(ref mut path) = hostos_icos_settings.ssh_authorized_keys_path { - *path = hostos_config_path.join("ssh_authorized_keys"); - } - let hostos_config = HostOSConfig { network_settings: setupos_config.network_settings, - icos_settings: hostos_icos_settings, + icos_settings: setupos_config.icos_settings, hostos_settings: setupos_config.hostos_settings, guestos_settings: setupos_config.guestos_settings, }; @@ -293,13 +284,6 @@ pub fn main() -> Result<()> { let hostos_config: HostOSConfig = serde_json::from_reader(File::open(hostos_config_json_path)?)?; - // update select file paths for GuestOS - let mut guestos_icos_settings = hostos_config.icos_settings; - let guestos_config_path = Path::new("/boot/config"); - if let Some(ref mut path) = guestos_icos_settings.ssh_authorized_keys_path { - *path = guestos_config_path.join("accounts_ssh_authorized_keys"); - } - // TODO: We won't have to modify networking between the hostos and // guestos config after completing the networking revamp (NODE-1327) let mut guestos_network_settings = hostos_config.network_settings; @@ -320,7 +304,7 @@ pub fn main() -> Result<()> { let guestos_config = GuestOSConfig { network_settings: guestos_network_settings, - icos_settings: guestos_icos_settings, + icos_settings: hostos_config.icos_settings, guestos_settings: hostos_config.guestos_settings, }; @@ -354,7 +338,7 @@ pub fn main() -> Result<()> { nns_public_key_exists: clap_args.nns_public_key_exists, nns_urls: clap_args.nns_urls, node_operator_private_key_exists: clap_args.node_operator_private_key_exists, - ssh_authorized_keys_path: clap_args.ssh_authorized_keys_path, + use_ssh_authorized_keys: clap_args.use_ssh_authorized_keys, inject_ic_crypto: clap_args.inject_ic_crypto, inject_ic_state: clap_args.inject_ic_state, inject_ic_registry_local_store: clap_args.inject_ic_registry_local_store, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index f9c4c8114eb..1b1832d66a4 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -2,7 +2,6 @@ use ic_types::malicious_behaviour::MaliciousBehaviour; use mac_address::mac_address::FormattedMacAddress; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, Ipv6Addr}; -use std::path::PathBuf; use url::Url; /// SetupOS configuration. User-facing configuration files @@ -96,13 +95,14 @@ pub struct ICOSSettings { /// The URL (HTTP) of the NNS node(s). pub nns_urls: Vec, pub node_operator_private_key_exists: bool, - /// This directory contains individual files named `admin`, `backup`, `readonly`. + /// This ssh keys directory contains individual files named `admin`, `backup`, `readonly`. /// The contents of these files serve as `authorized_keys` for their respective role account. /// This means that, for example, `accounts_ssh_authorized_keys/admin` /// is transferred to `~admin/.ssh/authorized_keys` on the target system. /// backup and readonly can only be modified via an NNS proposal /// and are in place for subnet recovery or issue debugging purposes. - pub ssh_authorized_keys_path: Option, + /// use_ssh_authorized_keys triggers the use of the ssh keys directory + pub use_ssh_authorized_keys: bool, pub icos_dev_settings: ICOSDevSettings, } diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index eef3a6d8521..83921b7fc54 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -221,7 +221,7 @@ fn main() { nns_public_key_exists: Some(true), nns_urls: Some(vec![format!("http://[{}]", ipv6_addr)]), node_operator_private_key_exists: Some(true), - ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), + use_ssh_authorized_keys: Some(true), inject_ic_crypto: Some(false), inject_ic_state: Some(false), inject_ic_registry_local_store: Some(false), diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index bcb121c3926..f85841db285 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -431,7 +431,7 @@ fn create_config_disk_image( nns_public_key_exists: Some(true), nns_urls: None, node_operator_private_key_exists: Some(true), - ssh_authorized_keys_path: Some("/boot/config/accounts_ssh_authorized_keys".into()), + use_ssh_authorized_keys: Some(true), inject_ic_crypto: Some(false), inject_ic_state: Some(false), inject_ic_registry_local_store: Some(false), From c6c7d678b72e447e20c58d8cf88f8086344ec17c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 28 Oct 2024 19:43:50 +0000 Subject: [PATCH 136/241] Uncomment setupos.sh scripts --- ic-os/components/setupos-scripts/setupos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/setupos-scripts/setupos.sh b/ic-os/components/setupos-scripts/setupos.sh index 82e6b096587..1c4d5645e58 100755 --- a/ic-os/components/setupos-scripts/setupos.sh +++ b/ic-os/components/setupos-scripts/setupos.sh @@ -40,8 +40,8 @@ main() { start_setupos /opt/ic/bin/check-setupos-age.sh /opt/ic/bin/check-config.sh - #/opt/ic/bin/check-hardware.sh - #/opt/ic/bin/check-network.sh + /opt/ic/bin/check-hardware.sh + /opt/ic/bin/check-network.sh /opt/ic/bin/setup-disk.sh /opt/ic/bin/install-hostos.sh /opt/ic/bin/install-guestos.sh From f7d1028929a1daed92544d41bac1acc7e0f05a64 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 28 Oct 2024 19:58:15 +0000 Subject: [PATCH 137/241] Fix reference to node_operator_private_key.pem --- ic-os/components/setupos-scripts/check-hardware.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/setupos-scripts/check-hardware.sh b/ic-os/components/setupos-scripts/check-hardware.sh index 3e77112fe75..eced389a306 100644 --- a/ic-os/components/setupos-scripts/check-hardware.sh +++ b/ic-os/components/setupos-scripts/check-hardware.sh @@ -249,7 +249,7 @@ function verify_disks() { function verify_deployment_path() { echo "* Verifying deployment path..." - if [[ ${GENERATION} == 2 ]] && [[ ! -f "/boot/config/node_operator_private_key.pem" ]]; then + if [[ ${GENERATION} == 2 ]] && [[ ! -f "${CONFIG_DIR}/node_operator_private_key.pem" ]]; then echo -e "\n\n\n\n\n\n" echo -e "\033[1;31mWARNING: Gen2 hardware detected but no Node Operator Private Key found.\033[0m" echo -e "\033[1;31mGen2 hardware should be deployed using the Gen2 Node Deployment method.\033[0m" From 1c21cc52535e81db59b8a9a04aadde2714017b90 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 28 Oct 2024 20:01:10 +0000 Subject: [PATCH 138/241] Fix reference to ssh_authorized_keys --- ic-os/components/setupos-scripts/setup-hostos-config.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 66f130d075d..efcc47dd8e6 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -23,9 +23,9 @@ function mount_config_partition() { function copy_config_files() { echo "* Copying SSH authorized keys..." use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') - if [[ "${ssh_authorized_keys,,}" == "true" ]]; then - if [ -d "${ssh_authorized_keys}" ]; then - cp -a "${ssh_authorized_keys}" /media/ + if [[ "${use_ssh_authorized_keys,,}" == "true" ]]; then + if [ -d "${CONFIG_DIR}/ssh_authorized_keys" ]; then + cp -a "${CONFIG_DIR}/ssh_authorized_keys" /media/ log_and_halt_installation_on_error "${?}" "Unable to copy SSH authorized keys to hostOS config partition." else echo >&2 "Warning: SSH authorized keys are not configured." From af1ad0700f2afca12ce0e5a6e1b0b219dee89ef8 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 29 Oct 2024 17:50:51 +0000 Subject: [PATCH 139/241] Convert ipv6_config_type to Enum --- .../config/src/generate_testnet_config.rs | 20 +++++++++++-------- rs/ic_os/config/src/main.rs | 6 ++++-- .../launch-single-vm/src/main.rs | 6 ++++-- rs/tests/driver/src/driver/bootstrap.rs | 8 +++++--- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 2d23d8d84fb..565ee8a89f8 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -9,7 +9,7 @@ use crate::types::*; pub struct GenerateTestnetConfigArgs { // NetworkSettings arguments - pub ipv6_config_type: Option, // "Deterministic", "Fixed", "RouterAdvertisement" + pub ipv6_config_type: Option, pub deterministic_prefix: Option, pub deterministic_prefix_length: Option, pub deterministic_gateway: Option, @@ -46,6 +46,13 @@ pub struct GenerateTestnetConfigArgs { pub hostname: Option, } +#[derive(Clone, clap::ValueEnum)] +pub enum Ipv6ConfigType { + Deterministic, + Fixed, + RouterAdvertisement, +} + /// Generates a writes a serialized GuestOSConfig to guestos_config_json_path /// Any required config fields that aren't specified will receive dummy values pub fn generate_testnet_config( @@ -85,8 +92,8 @@ pub fn generate_testnet_config( } = config; // Construct the NetworkSettings - let ipv6_config = match ipv6_config_type.as_deref() { - Some("Deterministic") => { + let ipv6_config = match ipv6_config_type { + Some(Ipv6ConfigType::Deterministic) => { let prefix = deterministic_prefix.ok_or_else(|| { anyhow::anyhow!( "deterministic_prefix is required when ipv6_config_type is 'Deterministic'" @@ -112,7 +119,7 @@ pub fn generate_testnet_config( gateway, }) } - Some("Fixed") => { + Some(Ipv6ConfigType::Fixed) => { let address = fixed_address.ok_or_else(|| { anyhow::anyhow!("fixed_address is required when ipv6_config_type is 'Fixed'") })?; @@ -126,10 +133,7 @@ pub fn generate_testnet_config( Ipv6Config::Fixed(FixedIpv6Config { address, gateway }) } // Default to RouterAdvertisement if not provided - Some("RouterAdvertisement") | None => Ipv6Config::RouterAdvertisement, - Some(other) => { - anyhow::bail!("Invalid ipv6_config_type '{}'. Must be 'Deterministic', 'Fixed', or 'RouterAdvertisement'.", other); - } + Some(Ipv6ConfigType::RouterAdvertisement) | None => Ipv6Config::RouterAdvertisement, }; let ipv4_config = match (ipv4_address, ipv4_gateway, ipv4_prefix_length, ipv4_domain) { diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index ad772a39291..45938b272e2 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -7,7 +7,9 @@ use mac_address::mac_address::{get_ipmi_mac, FormattedMacAddress}; use std::fs::File; use std::path::{Path, PathBuf}; -use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; +use config::generate_testnet_config::{ + generate_testnet_config, GenerateTestnetConfigArgs, Ipv6ConfigType, +}; use config::types::*; #[derive(Subcommand)] @@ -63,7 +65,7 @@ struct ConfigArgs { #[derive(Args)] pub struct GenerateTestnetConfigClapArgs { #[arg(long)] - pub ipv6_config_type: Option, // "Deterministic", "Fixed", "RouterAdvertisement" + pub ipv6_config_type: Option, #[arg(long)] pub deterministic_prefix: Option, #[arg(long)] diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index 83921b7fc54..8e35672d67b 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -23,7 +23,9 @@ use std::process::Command; use tempfile::tempdir; use url::Url; -use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; +use config::generate_testnet_config::{ + generate_testnet_config, GenerateTestnetConfigArgs, Ipv6ConfigType, +}; const FARM_BASE_URL: &str = "https://farm.dfinity.systems"; @@ -204,7 +206,7 @@ fn main() { // Build GuestOS config object let config = GenerateTestnetConfigArgs { - ipv6_config_type: Some("RouterAdvertisement".to_string()), + ipv6_config_type: Some(Ipv6ConfigType::RouterAdvertisement), deterministic_prefix: None, deterministic_prefix_length: None, deterministic_gateway: None, diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index f85841db285..6795916edbf 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -22,7 +22,9 @@ use crate::k8s::images::*; use crate::k8s::tnet::{TNet, TNode}; use crate::util::block_on; use anyhow::{bail, Result}; -use config::generate_testnet_config::{generate_testnet_config, GenerateTestnetConfigArgs}; +use config::generate_testnet_config::{ + generate_testnet_config, GenerateTestnetConfigArgs, Ipv6ConfigType, +}; use ic_base_types::NodeId; use ic_prep_lib::{ internet_computer::{IcConfig, InitializedIc, TopologyConfig}, @@ -414,7 +416,7 @@ fn create_config_disk_image( ) -> anyhow::Result<()> { // Build GuestOS config object let mut config = GenerateTestnetConfigArgs { - ipv6_config_type: Some("RouterAdvertisement".to_string()), + ipv6_config_type: Some(Ipv6ConfigType::RouterAdvertisement), deterministic_prefix: None, deterministic_prefix_length: None, deterministic_gateway: None, @@ -448,7 +450,7 @@ fn create_config_disk_image( // We've seen k8s nodes fail to pick up RA correctly, so we specify their // addresses directly. Ideally, all nodes should do this, to match mainnet. if InfraProvider::read_attribute(test_env) == InfraProvider::K8s { - config.ipv6_config_type = Some("Fixed".to_string()); + config.ipv6_config_type = Some(Ipv6ConfigType::Fixed); config.fixed_address = Some(format!("{}/64", node.node_config.public_api.ip())); config.fixed_gateway = Some("fe80::ecee:eeff:feee:eeee".to_string()); } From 3148be64c05dae10b8784ac5986c139d1a570544 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 29 Oct 2024 18:08:48 +0000 Subject: [PATCH 140/241] Add generate_testnet_config validation tests --- .../config/src/generate_testnet_config.rs | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 565ee8a89f8..d77db64dd90 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -7,6 +7,7 @@ use url::Url; use crate::serialize_and_write_config; use crate::types::*; +#[derive(Default)] pub struct GenerateTestnetConfigArgs { // NetworkSettings arguments pub ipv6_config_type: Option, @@ -252,3 +253,199 @@ pub fn generate_testnet_config( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_valid_configuration() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::RouterAdvertisement), + mgmt_mac: Some("00:11:22:33:44:55".to_string()), + nns_urls: Some(vec!["https://example.com".to_string()]), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/tmp/guestos_config.json")); + assert!(result.is_ok()); + } + + #[test] + fn test_missing_deterministic_prefix() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::Deterministic), + deterministic_prefix: None, + deterministic_prefix_length: Some(64), + deterministic_gateway: Some("fe80::1".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "deterministic_prefix is required when ipv6_config_type is 'Deterministic'" + ); + } + + #[test] + fn test_missing_deterministic_prefix_length() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::Deterministic), + deterministic_prefix: Some("2001:db8::".to_string()), + deterministic_prefix_length: None, + deterministic_gateway: Some("fe80::1".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "deterministic_prefix_length is required when ipv6_config_type is 'Deterministic'" + ); + } + + #[test] + fn test_missing_deterministic_gateway() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::Deterministic), + deterministic_prefix: Some("2001:db8::".to_string()), + deterministic_prefix_length: Some(64), + deterministic_gateway: None, + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "deterministic_gateway is required when ipv6_config_type is 'Deterministic'" + ); + } + + #[test] + fn test_invalid_deterministic_gateway() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::Deterministic), + deterministic_prefix: Some("2001:db8::".to_string()), + deterministic_prefix_length: Some(64), + deterministic_gateway: Some("invalid_ip".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Failed to parse deterministic_gateway")); + } + + #[test] + fn test_missing_fixed_address() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::Fixed), + fixed_address: None, + fixed_gateway: Some("fe80::1".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "fixed_address is required when ipv6_config_type is 'Fixed'" + ); + } + + #[test] + fn test_missing_fixed_gateway() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::Fixed), + fixed_address: Some("2001:db8::1/64".to_string()), + fixed_gateway: None, + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "fixed_gateway is required when ipv6_config_type is 'Fixed'" + ); + } + + #[test] + fn test_invalid_fixed_gateway() { + let args = GenerateTestnetConfigArgs { + ipv6_config_type: Some(Ipv6ConfigType::Fixed), + fixed_address: Some("2001:db8::1/64".to_string()), + fixed_gateway: Some("invalid_ip".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Failed to parse fixed_gateway")); + } + + #[test] + fn test_incomplete_ipv4_config() { + let args = GenerateTestnetConfigArgs { + ipv4_address: Some("192.0.2.1".to_string()), + ipv4_gateway: Some("192.0.2.254".to_string()), + ipv4_prefix_length: None, + ipv4_domain: Some("example.com".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Incomplete IPv4 configuration provided. All parameters (ipv4_address, ipv4_gateway, ipv4_prefix_length, ipv4_domain) are required for IPv4 configuration." + ); + } + + #[test] + fn test_invalid_ipv4_address() { + let args = GenerateTestnetConfigArgs { + ipv4_address: Some("invalid_ip".to_string()), + ipv4_gateway: Some("192.0.2.254".to_string()), + ipv4_prefix_length: Some(24), + ipv4_domain: Some("example.com".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Failed to parse ipv4_address")); + } + + #[test] + fn test_invalid_ipv4_gateway() { + let args = GenerateTestnetConfigArgs { + ipv4_address: Some("192.0.2.1".to_string()), + ipv4_gateway: Some("invalid_ip".to_string()), + ipv4_prefix_length: Some(24), + ipv4_domain: Some("example.com".to_string()), + ..Default::default() + }; + + let result = generate_testnet_config(args, PathBuf::from("/dev/null")); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Failed to parse ipv4_gateway")); + } +} From 74f6eb71d28325125e3cae07e2d9f7e6e0b674ce Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 29 Oct 2024 18:14:36 +0000 Subject: [PATCH 141/241] Expand comment for GenerateTestnetConfig --- rs/ic_os/config/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 45938b272e2..13c488ce689 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -51,7 +51,7 @@ pub enum Commands { #[arg(long, value_name = "ipv6_address")] guestos_ipv6_address: String, }, - /// Creates a GuestOSConfig object directly from GenerateTestnetConfigClapArgs + /// Creates a GuestOSConfig object directly from GenerateTestnetConfigClapArgs. Only used for testing purposes. GenerateTestnetConfig(GenerateTestnetConfigClapArgs), } From a9135de8192e3c8595a09e5d4b444d7a90a8bf1c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 30 Oct 2024 17:04:53 +0000 Subject: [PATCH 142/241] Add CONFIG_VERSION to track versions backwards compatibility --- .../config/src/generate_testnet_config.rs | 1 + rs/ic_os/config/src/lib.rs | 126 ++++++++++++++++++ rs/ic_os/config/src/main.rs | 1 + rs/ic_os/config/src/types.rs | 58 ++++---- 4 files changed, 159 insertions(+), 27 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index d77db64dd90..d5d70887644 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -190,6 +190,7 @@ pub fn generate_testnet_config( let use_ssh_authorized_keys = use_ssh_authorized_keys.unwrap_or(true); let icos_settings = ICOSSettings { + config_version: CONFIG_VERSION.to_string(), mgmt_mac, deployment_environment, logging, diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index e4a519427b1..557786bafa3 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -71,6 +71,7 @@ mod tests { }; let icos_dev_settings = ICOSDevSettings::default(); let icos_settings = ICOSSettings { + config_version: CONFIG_VERSION.to_string(), mgmt_mac: FormattedMacAddress::try_from("ec:2a:72:31:a2:0c")?, deployment_environment: "Mainnet".to_string(), logging, @@ -135,4 +136,129 @@ mod tests { Ok(()) } + + // Test config version 1.0.0 + const HOSTOS_CONFIG_JSON_V1_0_0: &str = r#" + { + "network_settings": { + "ipv6_config": { + "Deterministic": { + "prefix": "2a00:fb01:400:200", + "prefix_length": 64, + "gateway": "2a00:fb01:400:200::1" + } + }, + "ipv4_config": { + "address": "192.168.0.2", + "gateway": "192.168.0.1", + "prefix_length": 24, + "domain": "example.com" + } + }, + "icos_settings": { + "config_version": "1.0.0", + "mgmt_mac": "ec:2a:72:31:a2:0c", + "deployment_environment": "Mainnet", + "logging": { + "elasticsearch_hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443", + "elasticsearch_tags": "tag1 tag2" + }, + "nns_public_key_exists": true, + "nns_urls": [ + "http://localhost" + ], + "node_operator_private_key_exists": true, + "use_ssh_authorized_keys": false, + "icos_dev_settings": {} + }, + "hostos_settings": { + "vm_memory": 490, + "vm_cpu": "kvm", + "verbose": false + }, + "guestos_settings": { + "inject_ic_crypto": false, + "inject_ic_state": false, + "inject_ic_registry_local_store": false, + "guestos_dev_settings": { + "backup_spool": { + "backup_retention_time_seconds": 3600, + "backup_purging_interval_seconds": 600 + }, + "malicious_behavior": null, + "query_stats_epoch_length": 1000, + "bitcoind_addr": "127.0.0.1:8333", + "jaeger_addr": "127.0.0.1:6831", + "socks_proxy": "127.0.0.1:1080", + "hostname": "my-node" + } + } + } + "#; + + const GUESTOS_CONFIG_JSON_V1_0_0: &str = r#" + { + "network_settings": { + "ipv6_config": { + "Fixed": { + "address": "2a00:fb01:400:200::2/64", + "gateway": "2a00:fb01:400:200::1" + } + }, + "ipv4_config": null + }, + "icos_settings": { + "config_version": "1.0.0", + "mgmt_mac": "ec:2a:72:31:a2:0c", + "deployment_environment": "Mainnet", + "logging": { + "elasticsearch_hosts": "elasticsearch-node-0.mercury.dfinity.systems:443", + "elasticsearch_tags": "tag1 tag2" + }, + "nns_public_key_exists": true, + "nns_urls": [ + "http://localhost" + ], + "node_operator_private_key_exists": true, + "use_ssh_authorized_keys": false, + "icos_dev_settings": {} + }, + "guestos_settings": { + "inject_ic_crypto": true, + "inject_ic_state": true, + "inject_ic_registry_local_store": true, + "guestos_dev_settings": { + "backup_spool": { + "backup_retention_time_seconds": 7200, + "backup_purging_interval_seconds": 1200 + }, + "malicious_behavior": null, + "query_stats_epoch_length": 2000, + "bitcoind_addr": "127.0.0.1:8333", + "jaeger_addr": "127.0.0.1:6831", + "socks_proxy": "127.0.0.1:1080", + "hostname": "guest-node" + } + } + } + "#; + + #[test] + fn test_deserialize_hostos_config_v1_0_0() -> Result<(), Box> { + let config: HostOSConfig = serde_json::from_str(HOSTOS_CONFIG_JSON_V1_0_0)?; + assert_eq!(config.icos_settings.config_version, "1.0.0"); + assert_eq!(config.hostos_settings.vm_cpu, "kvm"); + Ok(()) + } + + #[test] + fn test_deserialize_guestos_config_v1_0_0() -> Result<(), Box> { + let config: GuestOSConfig = serde_json::from_str(GUESTOS_CONFIG_JSON_V1_0_0)?; + assert_eq!(config.icos_settings.config_version, "1.0.0"); + assert_eq!( + config.icos_settings.mgmt_mac.to_string(), + "ec:2a:72:31:a2:0c" + ); + Ok(()) + } } diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 13c488ce689..668d1634b4b 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -207,6 +207,7 @@ pub fn main() -> Result<()> { }; let icos_settings = ICOSSettings { + config_version: CONFIG_VERSION.to_string(), mgmt_mac, deployment_environment: deployment_json_settings.deployment.name, logging, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 1b1832d66a4..b2001546655 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, Ipv6Addr}; use url::Url; +pub const CONFIG_VERSION: &str = "1.0.0"; + /// SetupOS configuration. User-facing configuration files /// (e.g., `config.ini`, `deployment.json`) are transformed into `SetupOSConfig`. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] @@ -32,6 +34,34 @@ pub struct GuestOSConfig { pub guestos_settings: GuestOSSettings, } +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +pub struct ICOSSettings { + /// Tracks the config version, set to CONFIG_VERSION at runtime. + pub config_version: String, + /// In nested testing, mgmt_mac is set in deployment.json.template, + /// else found dynamically in call to config tool CreateSetuposConfig + pub mgmt_mac: FormattedMacAddress, + /// "mainnet" or "testnet" + pub deployment_environment: String, + pub logging: Logging, + pub nns_public_key_exists: bool, + /// The URL (HTTP) of the NNS node(s). + pub nns_urls: Vec, + pub node_operator_private_key_exists: bool, + /// This ssh keys directory contains individual files named `admin`, `backup`, `readonly`. + /// The contents of these files serve as `authorized_keys` for their respective role account. + /// This means that, for example, `accounts_ssh_authorized_keys/admin` + /// is transferred to `~admin/.ssh/authorized_keys` on the target system. + /// backup and readonly can only be modified via an NNS proposal + /// and are in place for subnet recovery or issue debugging purposes. + /// use_ssh_authorized_keys triggers the use of the ssh keys directory + pub use_ssh_authorized_keys: bool, + pub icos_dev_settings: ICOSDevSettings, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Default)] +pub struct ICOSDevSettings {} + /// Placeholder for SetupOS-specific settings. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct SetupOSSettings; @@ -83,32 +113,6 @@ pub struct BackupSpoolSettings { pub backup_purging_interval_seconds: Option, } -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] -pub struct ICOSSettings { - /// in nested testing, mgmt_mac is set in deployment.json.template, - /// else found dynamically in call to config tool CreateSetuposConfig - pub mgmt_mac: FormattedMacAddress, - /// "mainnet" or "testnet" - pub deployment_environment: String, - pub logging: Logging, - pub nns_public_key_exists: bool, - /// The URL (HTTP) of the NNS node(s). - pub nns_urls: Vec, - pub node_operator_private_key_exists: bool, - /// This ssh keys directory contains individual files named `admin`, `backup`, `readonly`. - /// The contents of these files serve as `authorized_keys` for their respective role account. - /// This means that, for example, `accounts_ssh_authorized_keys/admin` - /// is transferred to `~admin/.ssh/authorized_keys` on the target system. - /// backup and readonly can only be modified via an NNS proposal - /// and are in place for subnet recovery or issue debugging purposes. - /// use_ssh_authorized_keys triggers the use of the ssh keys directory - pub use_ssh_authorized_keys: bool, - pub icos_dev_settings: ICOSDevSettings, -} - -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Default)] -pub struct ICOSDevSettings {} - #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct Logging { /// Space-separated lists of hosts to ship logs to. @@ -147,7 +151,7 @@ pub struct DeterministicIpv6Config { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct FixedIpv6Config { - // fixed ipv6 address includes subnet mask /64 + // Fixed ipv6 address includes subnet mask /64 pub address: String, pub gateway: Ipv6Addr, } From 6830e55ebbdd13454fc69e5fbaa1cd40a56f9e13 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 30 Oct 2024 17:10:35 +0000 Subject: [PATCH 143/241] Update config unit test --- rs/ic_os/config/src/lib.rs | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 557786bafa3..1d650f05bfd 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -200,19 +200,25 @@ mod tests { { "network_settings": { "ipv6_config": { - "Fixed": { - "address": "2a00:fb01:400:200::2/64", + "Deterministic": { + "prefix": "2a00:fb01:400:200", + "prefix_length": 64, "gateway": "2a00:fb01:400:200::1" } }, - "ipv4_config": null + "ipv4_config": { + "address": "192.168.0.2", + "gateway": "192.168.0.1", + "prefix_length": 24, + "domain": "example.com" + } }, "icos_settings": { "config_version": "1.0.0", "mgmt_mac": "ec:2a:72:31:a2:0c", "deployment_environment": "Mainnet", "logging": { - "elasticsearch_hosts": "elasticsearch-node-0.mercury.dfinity.systems:443", + "elasticsearch_hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443", "elasticsearch_tags": "tag1 tag2" }, "nns_public_key_exists": true, @@ -224,20 +230,20 @@ mod tests { "icos_dev_settings": {} }, "guestos_settings": { - "inject_ic_crypto": true, - "inject_ic_state": true, - "inject_ic_registry_local_store": true, + "inject_ic_crypto": false, + "inject_ic_state": false, + "inject_ic_registry_local_store": false, "guestos_dev_settings": { "backup_spool": { - "backup_retention_time_seconds": 7200, - "backup_purging_interval_seconds": 1200 + "backup_retention_time_seconds": 3600, + "backup_purging_interval_seconds": 600 }, "malicious_behavior": null, - "query_stats_epoch_length": 2000, + "query_stats_epoch_length": 1000, "bitcoind_addr": "127.0.0.1:8333", "jaeger_addr": "127.0.0.1:6831", "socks_proxy": "127.0.0.1:1080", - "hostname": "guest-node" + "hostname": "my-node" } } } From 2921ed1a3a8f9fa288365e3d0d301eda4287013f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 30 Oct 2024 17:34:03 +0000 Subject: [PATCH 144/241] Add Configuration Update Protocol --- rs/ic_os/config/README.md | 5 ++++- rs/ic_os/config/src/types.rs | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/rs/ic_os/config/README.md b/rs/ic_os/config/README.md index 53567a8c9f6..61628086964 100644 --- a/rs/ic_os/config/README.md +++ b/rs/ic_os/config/README.md @@ -6,6 +6,9 @@ SetupOS transforms user-facing configuration files (like `config.ini`, `deployme All access to configuration and the config partition should go through the config structures. -For testing, IC-OS Config is also used to create HostOS and GuestOS configuration directly. +For testing, IC-OS Config is also used to create GuestOS configuration directly. + +When updating the IC-OS configuration, it's crucial to ensure backwards compatibility. +For detailed guidelines on updating the configuration, please refer to the documentation in [`types.rs`](src/types.rs). For details on the IC-OS configuration mechanism, refer to [ic-os/docs/Configuration.adoc](../../../ic-os/docs/Configuration.adoc) \ No newline at end of file diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index b2001546655..cc355a5809f 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -1,3 +1,19 @@ +//! # Configuration Update Protocol +//! +//! When updating the IC-OS configuration, it's crucial to maintain backwards compatibility. +//! Please adhere to the following guidelines when making changes to the configuration structures: +//! +//! - **Backwards Compatibility**: All updates should be backwards compatible to ensure that older configuration files are still deserializable. +//! +//! - **Updating `CONFIG_VERSION`**: Always update the `CONFIG_VERSION` constant (increment the minor version) whenever you modify the configuration. +//! +//! - **Unit Tests**: Add a unit test in `lib.rs` that tests deserialization of your new configuration version. +//! +//! - **Adding New Fields**: If adding a new field to a configuration struct, make sure it is optional or has a default value by implementing `Default` or via `#[serde(default)]`. +//! +//! - **Removing Fields**: If removing a field, ensure all references to it in the IC-OS codebase are eliminated. +//! +//! - **Renaming Fields**: Avoid renaming fields unless absolutely necessary. If you must rename a field, use `#[serde(rename = "old_name")]`. use ic_types::malicious_behaviour::MaliciousBehaviour; use mac_address::mac_address::FormattedMacAddress; use serde::{Deserialize, Serialize}; From 02c5e3e129429f179b4c80875c6cc5c5882bc789 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 30 Oct 2024 18:05:09 +0000 Subject: [PATCH 145/241] Update types.rs documentation --- rs/ic_os/config/src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index cc355a5809f..02b7968fd45 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -3,7 +3,7 @@ //! When updating the IC-OS configuration, it's crucial to maintain backwards compatibility. //! Please adhere to the following guidelines when making changes to the configuration structures: //! -//! - **Backwards Compatibility**: All updates should be backwards compatible to ensure that older configuration files are still deserializable. +//! - **Backwards Compatibility**: Configuration persists across reboots, so all config updates should be backwards compatible to ensure that older configuration files are still deserializable across GuestOS and HostOS upgrades. //! //! - **Updating `CONFIG_VERSION`**: Always update the `CONFIG_VERSION` constant (increment the minor version) whenever you modify the configuration. //! From 16ff02fe51d33a296008fd0227263af49fd4b47a Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 30 Oct 2024 19:34:04 +0000 Subject: [PATCH 146/241] Remove unnecessary deployment.template unit tests --- rs/ic_os/config/src/deployment_json.rs | 243 +------------------------ 1 file changed, 6 insertions(+), 237 deletions(-) diff --git a/rs/ic_os/config/src/deployment_json.rs b/rs/ic_os/config/src/deployment_json.rs index 94c411f4068..f3b9dbbf942 100644 --- a/rs/ic_os/config/src/deployment_json.rs +++ b/rs/ic_os/config/src/deployment_json.rs @@ -92,7 +92,7 @@ mod test { "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" }, "nns": { - "url": "https://wiki.internetcomputer.org/" + "url": "https://icp-api.io,https://icp0.io,https://ic0.app" }, "resources": { "memory": "490", @@ -111,7 +111,7 @@ mod test { "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" }, "nns": { - "url": "https://wiki.internetcomputer.org/" + "url": "https://icp-api.io,https://icp0.io,https://ic0.app" }, "resources": { "memory": "490", @@ -120,179 +120,6 @@ mod test { }"#; static DEPLOYMENT_STRUCT: Lazy = Lazy::new(|| { - let hosts = [ - "elasticsearch-node-0.mercury.dfinity.systems:443", - "elasticsearch-node-1.mercury.dfinity.systems:443", - "elasticsearch-node-2.mercury.dfinity.systems:443", - "elasticsearch-node-3.mercury.dfinity.systems:443", - ] - .join(" "); - DeploymentSettings { - deployment: Deployment { - name: "mainnet".to_string(), - mgmt_mac: None, - }, - logging: Logging { hosts }, - nns: Nns { - url: vec![Url::parse("https://wiki.internetcomputer.org").unwrap()], - }, - resources: Resources { - memory: 490, - cpu: Some("kvm".to_string()), - }, - } - }); - - const DEPLOYMENT_STR_NO_MGMT_MAC: &str = r#"{ - "deployment": { - "name": "mainnet" - }, - "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" - }, - "nns": { - "url": "https://wiki.internetcomputer.org/" - }, - "resources": { - "memory": "490", - "cpu": "kvm" - } -}"#; - - static DEPLOYMENT_STRUCT_NO_MGMT_MAC: Lazy = Lazy::new(|| { - let hosts = [ - "elasticsearch-node-0.mercury.dfinity.systems:443", - "elasticsearch-node-1.mercury.dfinity.systems:443", - "elasticsearch-node-2.mercury.dfinity.systems:443", - "elasticsearch-node-3.mercury.dfinity.systems:443", - ] - .join(" "); - DeploymentSettings { - deployment: Deployment { - name: "mainnet".to_string(), - mgmt_mac: None, - }, - logging: Logging { hosts }, - nns: Nns { - url: vec![Url::parse("https://wiki.internetcomputer.org").unwrap()], - }, - resources: Resources { - memory: 490, - cpu: Some("kvm".to_string()), - }, - } - }); - - const DEPLOYMENT_STR_NO_CPU_NO_MGMT_MAC: &str = r#"{ - "deployment": { - "name": "mainnet" - }, - "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" - }, - "nns": { - "url": "https://wiki.internetcomputer.org/" - }, - "resources": { - "memory": "490" - } -}"#; - - static DEPLOYMENT_STRUCT_NO_CPU_NO_MGMT_MAC: Lazy = Lazy::new(|| { - let hosts = [ - "elasticsearch-node-0.mercury.dfinity.systems:443", - "elasticsearch-node-1.mercury.dfinity.systems:443", - "elasticsearch-node-2.mercury.dfinity.systems:443", - "elasticsearch-node-3.mercury.dfinity.systems:443", - ] - .join(" "); - DeploymentSettings { - deployment: Deployment { - name: "mainnet".to_string(), - mgmt_mac: None, - }, - logging: Logging { hosts }, - nns: Nns { - url: vec![Url::parse("https://wiki.internetcomputer.org").unwrap()], - }, - resources: Resources { - memory: 490, - cpu: None, - }, - } - }); - - const QEMU_CPU_DEPLOYMENT_STR: &str = r#"{ - "deployment": { - "name": "mainnet" - }, - "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" - }, - "nns": { - "url": "https://wiki.internetcomputer.org/" - }, - "resources": { - "memory": "490", - "cpu": "qemu" - } -}"#; - - static QEMU_CPU_DEPLOYMENT_STRUCT: Lazy = Lazy::new(|| { - let hosts = [ - "elasticsearch-node-0.mercury.dfinity.systems:443", - "elasticsearch-node-1.mercury.dfinity.systems:443", - "elasticsearch-node-2.mercury.dfinity.systems:443", - "elasticsearch-node-3.mercury.dfinity.systems:443", - ] - .join(" "); - DeploymentSettings { - deployment: Deployment { - name: "mainnet".to_string(), - mgmt_mac: None, - }, - logging: Logging { hosts }, - nns: Nns { - url: vec![Url::parse("https://wiki.internetcomputer.org").unwrap()], - }, - resources: Resources { - memory: 490, - cpu: Some("qemu".to_string()), - }, - } - }); - - const MULTI_URL_STR: &str = r#"{ - "deployment": { - "name": "mainnet" - }, - "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" - }, - "nns": { - "url": "http://[2001:920:401a:1710:5000:6aff:fee4:19cd]:8080/,http://[2600:3006:1400:1500:5000:19ff:fe38:c418]:8080/,http://[2600:2c01:21:0:5000:27ff:fe23:4839]:8080/" - }, - "resources": { - "memory": "490" - } -}"#; - - const MULTI_URL_SANS_SLASH_STR: &str = r#"{ - "deployment": { - "name": "mainnet" - }, - "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" - }, - "nns": { - "url": "http://[2001:920:401a:1710:5000:6aff:fee4:19cd]:8080,http://[2600:3006:1400:1500:5000:19ff:fe38:c418]:8080,http://[2600:2c01:21:0:5000:27ff:fe23:4839]:8080" - }, - "resources": { - "memory": "490" - } -}"#; - - static MULTI_URL_STRUCT: Lazy = Lazy::new(|| { let hosts = [ "elasticsearch-node-0.mercury.dfinity.systems:443", "elasticsearch-node-1.mercury.dfinity.systems:443", @@ -308,14 +135,14 @@ mod test { logging: Logging { hosts }, nns: Nns { url: vec![ - Url::parse("http://[2001:920:401a:1710:5000:6aff:fee4:19cd]:8080").unwrap(), - Url::parse("http://[2600:3006:1400:1500:5000:19ff:fe38:c418]:8080").unwrap(), - Url::parse("http://[2600:2c01:21:0:5000:27ff:fe23:4839]:8080").unwrap(), + Url::parse("https://icp-api.io").unwrap(), + Url::parse("https://icp0.io").unwrap(), + Url::parse("https://ic0.app").unwrap(), ], }, resources: Resources { memory: 490, - cpu: None, + cpu: Some("kvm".to_string()), }, } }); @@ -326,32 +153,6 @@ mod test { assert_eq!(*DEPLOYMENT_STRUCT, parsed_deployment); - let parsed_deployment = { serde_json::from_str(DEPLOYMENT_STR_NO_MGMT_MAC).unwrap() }; - - assert_eq!(*DEPLOYMENT_STRUCT_NO_MGMT_MAC, parsed_deployment); - - let parsed_deployment = - { serde_json::from_str(DEPLOYMENT_STR_NO_CPU_NO_MGMT_MAC).unwrap() }; - - assert_eq!(*DEPLOYMENT_STRUCT_NO_CPU_NO_MGMT_MAC, parsed_deployment); - - let parsed_cpu_deployment = { serde_json::from_str(QEMU_CPU_DEPLOYMENT_STR).unwrap() }; - - assert_eq!(*QEMU_CPU_DEPLOYMENT_STRUCT, parsed_cpu_deployment); - - let parsed_multi_url_deployment = { serde_json::from_str(MULTI_URL_STR).unwrap() }; - - assert_eq!(*MULTI_URL_STRUCT, parsed_multi_url_deployment); - - // NOTE: Canonically, url thinks these addresses should have a trailing - // slash, so the above case parses with a slash for the sake of the - // writeback test below. In practice, we have used addresses without - // this slash, so here we verify that this parses to the same value. - let parsed_multi_url_sans_slash_deployment = - { serde_json::from_str(MULTI_URL_SANS_SLASH_STR).unwrap() }; - - assert_eq!(*MULTI_URL_STRUCT, parsed_multi_url_sans_slash_deployment); - // Exercise DeserializeOwned using serde_json::from_value. // DeserializeOwned is used by serde_json::from_reader, which is the // main entrypoint of this code, in practice. @@ -359,36 +160,4 @@ mod test { assert_eq!(*DEPLOYMENT_STRUCT, parsed_deployment); } - - #[test] - fn serialize_deployment() { - let serialized_deployment = serde_json::to_string_pretty(&*DEPLOYMENT_STRUCT).unwrap(); - - // DEPLOYMENT_STRUCT serializes to DEPLOYMENT_STR_NO_MGMT_MAC because mgmt_mac field is skipped in serialization - assert_eq!(DEPLOYMENT_STR_NO_MGMT_MAC, serialized_deployment); - - let serialized_deployment = - serde_json::to_string_pretty(&*DEPLOYMENT_STRUCT_NO_CPU_NO_MGMT_MAC).unwrap(); - - assert_eq!(DEPLOYMENT_STR_NO_CPU_NO_MGMT_MAC, serialized_deployment); - - let serialized_deployment = - serde_json::to_string_pretty(&*DEPLOYMENT_STRUCT_NO_MGMT_MAC).unwrap(); - - assert_eq!(DEPLOYMENT_STR_NO_MGMT_MAC, serialized_deployment); - - let serialized_deployment = - serde_json::to_string_pretty(&*DEPLOYMENT_STRUCT_NO_CPU_NO_MGMT_MAC).unwrap(); - - assert_eq!(DEPLOYMENT_STR_NO_CPU_NO_MGMT_MAC, serialized_deployment); - - let serialized_deployment = - serde_json::to_string_pretty(&*QEMU_CPU_DEPLOYMENT_STRUCT).unwrap(); - - assert_eq!(QEMU_CPU_DEPLOYMENT_STR, serialized_deployment); - - let serialized_deployment = serde_json::to_string_pretty(&*MULTI_URL_STRUCT).unwrap(); - - assert_eq!(MULTI_URL_STR, serialized_deployment); - } } From 718137665f2633f2a00023268c34d2e8158ef350 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 30 Oct 2024 20:47:25 +0000 Subject: [PATCH 147/241] Update configuration guidance --- ic-os/docs/Configuration.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index 70b1f2a6b29..d1b54b5f34f 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -65,7 +65,8 @@ In the absence of a sealing key (which will be available in SEV-protected truste === Guidance for adding configuration bits -TODO: update... +To add or modify a configuration field within the icos config tool, see +link:../../rs/ic_os/config/README.md[rs/ic_os/config]. To add a new configuration file/directory: From 8ca8583f6934467cce859f35bbf35f37819f8853 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 31 Oct 2024 19:02:58 +0000 Subject: [PATCH 148/241] Add generate_ic_boundary_tls_cert to GuestOSDevSettings --- ic-os/components/ic/generate-replica-config.sh | 11 +++++++++++ .../init/bootstrap-ic-node/bootstrap-ic-node.sh | 6 ------ rs/ic_os/config/src/generate_testnet_config.rs | 3 +++ rs/ic_os/config/src/main.rs | 3 +++ rs/ic_os/config/src/types.rs | 3 +++ rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs | 1 + rs/tests/driver/src/driver/bootstrap.rs | 3 ++- 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 06447b51539..5cf3c7c7cdb 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -26,6 +26,8 @@ function read_config_variables() { # Compact the JSON and escape special characters MALICIOUS_BEHAVIOR=$(get_config_value '.guestos_settings.guestos_dev_settings.malicious_behavior' | jq -c '.' | sed 's/[&\/]/\\&/g') + + GENERATE_IC_BOUNDARY_TLS_CERT=$(get_config_value '.guestos_settings.guestos_dev_settings.generate_ic_boundary_tls_cert') } function configure_ipv6() { @@ -155,6 +157,15 @@ sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ jaeger_addr }}@${JAEGER_ADDR}@" \ "${IN_FILE}" >"${OUT_FILE}" +# Generate and inject a self-signed TLS certificate and key for ic-boundary +# for the given domain name. To be used in system tests only. +if [[ -n "${GENERATE_IC_BOUNDARY_TLS_CERT}" ]] && [ "${GENERATE_IC_BOUNDARY_TLS_CERT}" != "null" ]; then + openssl req -x509 -newkey rsa:2048 \ + -keyout /var/lib/ic/ic-boundary-tls.key \ + -out /var/lib/ic/ic-boundary-tls.crt -sha256 -days 3650 -nodes \ + -subj /C=CH/ST=Zurich/L=Zurich/O=InternetComputer/OU=ApiBoundaryNodes/CN=${GENERATE_IC_BOUNDARY_TLS_CERT} +fi + # umask for service is set to be restricted, but this file needs to be # world-readable chmod 644 "${OUT_FILE}" diff --git a/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh index a10aea8de9e..7be1d64f502 100755 --- a/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh @@ -99,12 +99,6 @@ function process_bootstrap() { cp -rL -T "${TMPDIR}/${ITEM}" "${STATE_ROOT}/data/${ITEM}" fi done - if [ -e "${TMPDIR}/ic-boundary-tls.key" ]; then - echo "Setting up self-signed certificate of ic-boundary" - cp -L "${TMPDIR}/ic-boundary-tls.key" "${STATE_ROOT}/data/ic-boundary-tls.key" - cp -L "${TMPDIR}/ic-boundary-tls.crt" "${STATE_ROOT}/data/ic-boundary-tls.crt" - sudo chmod +r ${STATE_ROOT}/data/ic-boundary-tls.key - fi for FILE in config.json; do if [ -e "${TMPDIR}/${FILE}" ]; then diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index d5d70887644..66c434fd705 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -45,6 +45,7 @@ pub struct GenerateTestnetConfigArgs { pub jaeger_addr: Option, pub socks_proxy: Option, pub hostname: Option, + pub generate_ic_boundary_tls_cert: Option, } #[derive(Clone, clap::ValueEnum)] @@ -90,6 +91,7 @@ pub fn generate_testnet_config( jaeger_addr, socks_proxy, hostname, + generate_ic_boundary_tls_cert, } = config; // Construct the NetworkSettings @@ -226,6 +228,7 @@ pub fn generate_testnet_config( jaeger_addr, socks_proxy, hostname, + generate_ic_boundary_tls_cert, }; // Construct GuestOSSettings diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 668d1634b4b..d4a9b944a33 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -128,6 +128,8 @@ pub struct GenerateTestnetConfigClapArgs { pub socks_proxy: Option, #[arg(long)] pub hostname: Option, + #[arg(long)] + pub generate_ic_boundary_tls_cert: Option, // Output path #[arg(long)] @@ -353,6 +355,7 @@ pub fn main() -> Result<()> { jaeger_addr: clap_args.jaeger_addr, socks_proxy: clap_args.socks_proxy, hostname: clap_args.hostname, + generate_ic_boundary_tls_cert: clap_args.generate_ic_boundary_tls_cert, }; generate_testnet_config(args, clap_args.guestos_config_json_path) diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 02b7968fd45..bf5fa1636c0 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -118,6 +118,9 @@ pub struct GuestOSDevSettings { pub socks_proxy: Option, // An optional hostname to override the deterministically generated hostname pub hostname: Option, + // Generate and inject a self-signed TLS certificate and key for ic-boundary + // for the given domain name. To be used in system tests only. + pub generate_ic_boundary_tls_cert: Option, } /// Configures the usage of the backup spool directory. diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index 24c7c399c0e..cf461094216 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -236,6 +236,7 @@ fn main() { jaeger_addr: None, socks_proxy: None, hostname: None, + generate_ic_boundary_tls_cert: None, }; // populate guestos_config_json_path with serialized guestos config object diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 725f8494e5a..20bbe89d76c 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -454,6 +454,7 @@ fn create_config_disk_image( jaeger_addr: None, socks_proxy: None, hostname: None, + generate_ic_boundary_tls_cert: None, }; // We've seen k8s nodes fail to pick up RA correctly, so we specify their @@ -505,7 +506,7 @@ fn create_config_disk_image( // if the node has a domain name, generate a certificate to be used // when the node is an API boundary node. if let Some(domain_name) = &node.node_config.domain { - cmd.arg("--generate_ic_boundary_tls_cert").arg(domain_name); + config.generate_ic_boundary_tls_cert = Some(domain_name); } if let Some(domain) = domain { From 5c539039bb2363f19ab80e643ad213946a7e5e92 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 31 Oct 2024 19:07:42 +0000 Subject: [PATCH 149/241] Fix bootstrap.rs --- rs/tests/driver/src/driver/bootstrap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 20bbe89d76c..143c63d3ab2 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -506,7 +506,7 @@ fn create_config_disk_image( // if the node has a domain name, generate a certificate to be used // when the node is an API boundary node. if let Some(domain_name) = &node.node_config.domain { - config.generate_ic_boundary_tls_cert = Some(domain_name); + config.generate_ic_boundary_tls_cert = Some(domain_name.to_string()); } if let Some(domain) = domain { From be133ccf0204910e1da87ab362efa87c42c734c0 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 31 Oct 2024 19:14:48 +0000 Subject: [PATCH 150/241] Fix ic-boundary-tls output paths --- ic-os/components/ic/generate-replica-config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 5cf3c7c7cdb..d85404d95f8 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -161,8 +161,8 @@ sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ # for the given domain name. To be used in system tests only. if [[ -n "${GENERATE_IC_BOUNDARY_TLS_CERT}" ]] && [ "${GENERATE_IC_BOUNDARY_TLS_CERT}" != "null" ]; then openssl req -x509 -newkey rsa:2048 \ - -keyout /var/lib/ic/ic-boundary-tls.key \ - -out /var/lib/ic/ic-boundary-tls.crt -sha256 -days 3650 -nodes \ + -keyout /var/lib/ic/data/ic-boundary-tls.key \ + -out /var/lib/ic/data/ic-boundary-tls.crt -sha256 -days 3650 -nodes \ -subj /C=CH/ST=Zurich/L=Zurich/O=InternetComputer/OU=ApiBoundaryNodes/CN=${GENERATE_IC_BOUNDARY_TLS_CERT} fi From a725e937d1650a269b325cf021121b2e38f6ded0 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 1 Nov 2024 14:37:24 +0000 Subject: [PATCH 151/241] Add generate_ic_boundary_tls_cert to config unit test --- rs/ic_os/config/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 1d650f05bfd..8f717ab39c9 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -190,7 +190,8 @@ mod tests { "bitcoind_addr": "127.0.0.1:8333", "jaeger_addr": "127.0.0.1:6831", "socks_proxy": "127.0.0.1:1080", - "hostname": "my-node" + "hostname": "my-node", + "generate_ic_boundary_tls_cert": "domain" } } } @@ -243,7 +244,8 @@ mod tests { "bitcoind_addr": "127.0.0.1:8333", "jaeger_addr": "127.0.0.1:6831", "socks_proxy": "127.0.0.1:1080", - "hostname": "my-node" + "hostname": "my-node", + "generate_ic_boundary_tls_cert": "domain" } } } From b098aaca07638ee1661fe6596f859190eef956cc Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 1 Nov 2024 14:51:20 +0000 Subject: [PATCH 152/241] Uncouple domain_name from ipv4_config --- rs/ic_os/config/src/config_ini.rs | 11 +++++---- .../config/src/generate_testnet_config.rs | 19 +++++++-------- rs/ic_os/config/src/lib.rs | 13 +++++----- rs/ic_os/config/src/main.rs | 24 +++++++++---------- rs/ic_os/config/src/types.rs | 2 +- 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/rs/ic_os/config/src/config_ini.rs b/rs/ic_os/config/src/config_ini.rs index 518da5d5058..c37b8f9d178 100644 --- a/rs/ic_os/config/src/config_ini.rs +++ b/rs/ic_os/config/src/config_ini.rs @@ -15,7 +15,7 @@ pub struct ConfigIniSettings { pub ipv4_address: Option, pub ipv4_gateway: Option, pub ipv4_prefix_length: Option, - pub domain: Option, + pub domain_name: Option, pub verbose: bool, } @@ -82,7 +82,7 @@ pub fn get_config_ini_settings(config_file_path: &Path) -> Result Result()? ); assert_eq!(config_ini_settings.ipv4_prefix_length.unwrap(), 28); - assert_eq!(config_ini_settings.domain, Some("example.com".to_string())); + assert_eq!( + config_ini_settings.domain_name, + Some("example.com".to_string()) + ); assert!(!config_ini_settings.verbose); // Test missing ipv6 diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 66c434fd705..6085e7945c1 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -19,7 +19,7 @@ pub struct GenerateTestnetConfigArgs { pub ipv4_address: Option, pub ipv4_gateway: Option, pub ipv4_prefix_length: Option, - pub ipv4_domain: Option, + pub domain_name: Option, // ICOSSettings arguments pub mgmt_mac: Option, @@ -71,7 +71,7 @@ pub fn generate_testnet_config( ipv4_address, ipv4_gateway, ipv4_prefix_length, - ipv4_domain, + domain_name, mgmt_mac, deployment_environment, elasticsearch_hosts, @@ -139,8 +139,8 @@ pub fn generate_testnet_config( Some(Ipv6ConfigType::RouterAdvertisement) | None => Ipv6Config::RouterAdvertisement, }; - let ipv4_config = match (ipv4_address, ipv4_gateway, ipv4_prefix_length, ipv4_domain) { - (Some(addr_str), Some(gw_str), Some(prefix_len), Some(domain)) => Some(Ipv4Config { + let ipv4_config = match (ipv4_address, ipv4_gateway, ipv4_prefix_length) { + (Some(addr_str), Some(gw_str), Some(prefix_len)) => Some(Ipv4Config { address: addr_str .parse::() .map_err(|e| anyhow::anyhow!("Failed to parse ipv4_address: {}", e))?, @@ -148,17 +148,17 @@ pub fn generate_testnet_config( .parse::() .map_err(|e| anyhow::anyhow!("Failed to parse ipv4_gateway: {}", e))?, prefix_length: prefix_len, - domain, }), - (None, None, None, None) => None, + (None, None, None) => None, _ => { - anyhow::bail!("Incomplete IPv4 configuration provided. All parameters (ipv4_address, ipv4_gateway, ipv4_prefix_length, ipv4_domain) are required for IPv4 configuration."); + anyhow::bail!("Incomplete IPv4 configuration provided. All parameters (ipv4_address, ipv4_gateway, ipv4_prefix_length) are required for IPv4 configuration."); } }; let network_settings = NetworkSettings { ipv6_config, ipv4_config, + domain_name, }; // Construct ICOSSettings @@ -405,7 +405,6 @@ mod tests { ipv4_address: Some("192.0.2.1".to_string()), ipv4_gateway: Some("192.0.2.254".to_string()), ipv4_prefix_length: None, - ipv4_domain: Some("example.com".to_string()), ..Default::default() }; @@ -413,7 +412,7 @@ mod tests { assert!(result.is_err()); assert_eq!( result.unwrap_err().to_string(), - "Incomplete IPv4 configuration provided. All parameters (ipv4_address, ipv4_gateway, ipv4_prefix_length, ipv4_domain) are required for IPv4 configuration." + "Incomplete IPv4 configuration provided. All parameters (ipv4_address, ipv4_gateway, ipv4_prefix_length) are required for IPv4 configuration." ); } @@ -423,7 +422,6 @@ mod tests { ipv4_address: Some("invalid_ip".to_string()), ipv4_gateway: Some("192.0.2.254".to_string()), ipv4_prefix_length: Some(24), - ipv4_domain: Some("example.com".to_string()), ..Default::default() }; @@ -441,7 +439,6 @@ mod tests { ipv4_address: Some("192.0.2.1".to_string()), ipv4_gateway: Some("invalid_ip".to_string()), ipv4_prefix_length: Some(24), - ipv4_domain: Some("example.com".to_string()), ..Default::default() }; diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 8f717ab39c9..249c15036ac 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -58,6 +58,7 @@ mod tests { let network_settings = NetworkSettings { ipv6_config, ipv4_config: None, + domain_name: None, }; let logging = Logging { elasticsearch_hosts: [ @@ -151,9 +152,9 @@ mod tests { "ipv4_config": { "address": "192.168.0.2", "gateway": "192.168.0.1", - "prefix_length": 24, - "domain": "example.com" - } + "prefix_length": 24 + }, + "domain_name": "example.com" }, "icos_settings": { "config_version": "1.0.0", @@ -210,9 +211,9 @@ mod tests { "ipv4_config": { "address": "192.168.0.2", "gateway": "192.168.0.1", - "prefix_length": 24, - "domain": "example.com" - } + "prefix_length": 24 + }, + "domain_name": "example.com" }, "icos_settings": { "config_version": "1.0.0", diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index d4a9b944a33..b81e534b197 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -83,7 +83,7 @@ pub struct GenerateTestnetConfigClapArgs { #[arg(long)] pub ipv4_prefix_length: Option, #[arg(long)] - pub ipv4_domain: Option, + pub domain_name: Option, // ICOSSettings arguments #[arg(long)] @@ -156,7 +156,7 @@ pub fn main() -> Result<()> { ipv4_address, ipv4_gateway, ipv4_prefix_length, - domain, + domain_name, verbose, } = get_config_ini_settings(&config_ini_path)?; @@ -167,16 +167,13 @@ pub fn main() -> Result<()> { gateway: ipv6_gateway, }; - let ipv4_config = match (ipv4_address, ipv4_gateway, ipv4_prefix_length, domain) { - (Some(address), Some(gateway), Some(prefix_length), Some(domain)) => { - Some(Ipv4Config { - address, - gateway, - prefix_length, - domain, - }) - } - (None, None, None, None) => None, + let ipv4_config = match (ipv4_address, ipv4_gateway, ipv4_prefix_length) { + (Some(address), Some(gateway), Some(prefix_length)) => Some(Ipv4Config { + address, + gateway, + prefix_length, + }), + (None, None, None) => None, _ => { println!("Warning: Partial IPv4 configuration provided. All parameters are required for IPv4 configuration."); None @@ -186,6 +183,7 @@ pub fn main() -> Result<()> { let network_settings = NetworkSettings { ipv6_config: Ipv6Config::Deterministic(deterministic_config), ipv4_config, + domain_name, }; // get deployment.json variables @@ -335,7 +333,7 @@ pub fn main() -> Result<()> { ipv4_address: clap_args.ipv4_address, ipv4_gateway: clap_args.ipv4_gateway, ipv4_prefix_length: clap_args.ipv4_prefix_length, - ipv4_domain: clap_args.ipv4_domain, + domain_name: clap_args.domain_name, mgmt_mac: clap_args.mgmt_mac, deployment_environment: clap_args.deployment_environment, elasticsearch_hosts: clap_args.elasticsearch_hosts, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index bf5fa1636c0..65f918d8830 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -144,6 +144,7 @@ pub struct Logging { pub struct NetworkSettings { pub ipv6_config: Ipv6Config, pub ipv4_config: Option, + pub domain_name: Option, } #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] @@ -151,7 +152,6 @@ pub struct Ipv4Config { pub address: Ipv4Addr, pub gateway: Ipv4Addr, pub prefix_length: u8, - pub domain: String, } #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] From 3f398e84520bfdedf382e4feb10ae8b458ae82f3 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 1 Nov 2024 14:57:54 +0000 Subject: [PATCH 153/241] Update testing to fix calls to domain_name --- rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs | 2 +- rs/tests/driver/src/driver/bootstrap.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index cf461094216..d468a38e882 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -216,7 +216,7 @@ fn main() { ipv4_address: None, ipv4_gateway: None, ipv4_prefix_length: None, - ipv4_domain: None, + domain_name: None, mgmt_mac: None, deployment_environment: Some("testnet".to_string()), elasticsearch_hosts: None, diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 143c63d3ab2..ec049b00630 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -419,7 +419,7 @@ fn create_config_disk_image( malicious_behavior: Option, query_stats_epoch_length: Option, ipv4_config: Option, - domain: Option, + domain_name: Option, test_env: &TestEnv, group_name: &str, ) -> anyhow::Result<()> { @@ -434,7 +434,7 @@ fn create_config_disk_image( ipv4_address: None, ipv4_gateway: None, ipv4_prefix_length: None, - ipv4_domain: None, + domain_name: None, mgmt_mac: None, deployment_environment: Some("testnet".to_string()), elasticsearch_hosts: None, @@ -509,12 +509,12 @@ fn create_config_disk_image( config.generate_ic_boundary_tls_cert = Some(domain_name.to_string()); } - if let Some(domain) = domain { + if let Some(domain_name) = domain_name { info!( test_env.logger(), - "Node with id={} has domain_name {}", node.node_id, domain, + "Node with id={} has domain_name {}", node.node_id, domain_name, ); - config.ipv4_domain = Some(domain); + config.domain_name = Some(domain_name); } let elasticsearch_hosts: Vec = get_elasticsearch_hosts()?; From d22f2ac14882dec943571230cc59e87460a4d0ce Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 1 Nov 2024 16:52:36 +0000 Subject: [PATCH 154/241] Fix TLS_KEY_PATH and TLS_CERT_PATH file permissions --- ic-os/components/ic/generate-replica-config.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index d85404d95f8..2acb6e0c7b3 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -157,15 +157,19 @@ sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ jaeger_addr }}@${JAEGER_ADDR}@" \ "${IN_FILE}" >"${OUT_FILE}" +# umask for service is set to be restricted, but this file needs to be +# world-readable +chmod 644 "${OUT_FILE}" + # Generate and inject a self-signed TLS certificate and key for ic-boundary # for the given domain name. To be used in system tests only. if [[ -n "${GENERATE_IC_BOUNDARY_TLS_CERT}" ]] && [ "${GENERATE_IC_BOUNDARY_TLS_CERT}" != "null" ]; then + TLS_KEY_PATH="/var/lib/ic/data/ic-boundary-tls.key" + TLS_CERT_PATH="/var/lib/ic/data/ic-boundary-tls.crt" + openssl req -x509 -newkey rsa:2048 \ - -keyout /var/lib/ic/data/ic-boundary-tls.key \ - -out /var/lib/ic/data/ic-boundary-tls.crt -sha256 -days 3650 -nodes \ + -keyout "${TLS_KEY_PATH}" \ + -out "${TLS_CERT_PATH}" -sha256 -days 3650 -nodes \ -subj /C=CH/ST=Zurich/L=Zurich/O=InternetComputer/OU=ApiBoundaryNodes/CN=${GENERATE_IC_BOUNDARY_TLS_CERT} + chmod 644 "${TLS_KEY_PATH}" "${TLS_CERT_PATH}" fi - -# umask for service is set to be restricted, but this file needs to be -# world-readable -chmod 644 "${OUT_FILE}" From ef59cfc0012cc48ef88f5372f7d9aa0091d831e6 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 1 Nov 2024 21:07:25 +0000 Subject: [PATCH 155/241] Fix reference to config domain_name --- ic-os/components/ic/generate-replica-config.sh | 7 ++++--- ic-os/components/ic/ic.json5.template | 2 +- ic-os/components/setupos-scripts/check-network.sh | 12 +++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ic-os/components/ic/generate-replica-config.sh b/ic-os/components/ic/generate-replica-config.sh index 2acb6e0c7b3..4f0b538913f 100755 --- a/ic-os/components/ic/generate-replica-config.sh +++ b/ic-os/components/ic/generate-replica-config.sh @@ -23,6 +23,7 @@ function read_config_variables() { BACKUP_PURGING_INTERVAL_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_purging_interval_seconds') QUERY_STATS_EPOCH_LENGTH=$(get_config_value '.guestos_settings.guestos_dev_settings.query_stats_epoch_length') JAEGER_ADDR=$(get_config_value '.guestos_settings.guestos_dev_settings.jaeger_addr') + DOMAIN_NAME=$(get_config_value '.network_settings.domain_name') # Compact the JSON and escape special characters MALICIOUS_BEHAVIOR=$(get_config_value '.guestos_settings.guestos_dev_settings.malicious_behavior' | jq -c '.' | sed 's/[&\/]/\\&/g') @@ -59,14 +60,13 @@ function configure_ipv6() { } function configure_ipv4() { - IPV4_ADDRESS="" IPV4_GATEWAY="" DOMAIN="" + IPV4_ADDRESS="" IPV4_GATEWAY="" ipv4_config_present=$(get_config_value '.network_settings.ipv4_config != null') if [ "$ipv4_config_present" = "true" ]; then ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') IPV4_ADDRESS="${ipv4_address}/${ipv4_prefix_length}" IPV4_GATEWAY=$(get_config_value '.network_settings.ipv4_config.gateway') - DOMAIN=$(get_config_value '.network_settings.ipv4_config.domain') fi } @@ -113,6 +113,7 @@ function set_default_config_values() { [ "${BACKUP_PURGING_INTERVAL_SECS}" = "null" ] && BACKUP_PURGING_INTERVAL_SECS="3600" # Default is 1h [ "${QUERY_STATS_EPOCH_LENGTH}" = "null" ] && QUERY_STATS_EPOCH_LENGTH="600" # Default is 600 blocks (around 10min) [ "${JAEGER_ADDR}" = "null" ] && JAEGER_ADDR="" + [ "${DOMAIN_NAME}" = "null" ] && DOMAIN_NAME="" # todo: remove node_index variable and hard-code into ic.json5.template NODE_INDEX="0" @@ -147,7 +148,7 @@ set_default_config_values sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \ -e "s@{{ ipv4_gateway }}@${IPV4_GATEWAY}@" \ - -e "s@{{ domain }}@${DOMAIN}@" \ + -e "s@{{ domain_name }}@${DOMAIN_NAME}@" \ -e "s@{{ nns_urls }}@${NNS_URLS}@" \ -e "s@{{ node_index }}@${NODE_INDEX}@" \ -e "s@{{ backup_retention_time_secs }}@${BACKUP_RETENTION_TIME_SECS}@" \ diff --git a/ic-os/components/ic/ic.json5.template b/ic-os/components/ic/ic.json5.template index 757567e9fd3..4f2d69c80e2 100644 --- a/ic-os/components/ic/ic.json5.template +++ b/ic-os/components/ic/ic.json5.template @@ -24,7 +24,7 @@ // ============================================ // Configuration of the domain name // ============================================ - domain: "{{ domain }}", + domain: "{{ domain_name }}", // ============================================ // Configuration of registry client diff --git a/ic-os/components/setupos-scripts/check-network.sh b/ic-os/components/setupos-scripts/check-network.sh index fb65abad412..93b7f5cb3f7 100755 --- a/ic-os/components/setupos-scripts/check-network.sh +++ b/ic-os/components/setupos-scripts/check-network.sh @@ -15,7 +15,7 @@ function read_config_variables() { ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') - domain=$(get_config_value '.network_settings.ipv4_config.domain') + domain_name=$(get_config_value '.network_settings.domain_name') } # WARNING: Uses 'eval' for command execution. @@ -101,11 +101,13 @@ function print_network_settings() { echo "* Printing user defined network settings..." echo " IPv6 Prefix : ${ipv6_prefix}" echo " IPv6 Gateway: ${ipv6_gateway}" - if [[ -n ${ipv4_address} && -n ${ipv4_prefix_length} && -n ${ipv4_gateway} && -n ${domain} ]]; then + if [[ -n ${ipv4_address} && -n ${ipv4_prefix_length} && -n ${ipv4_gateway} ]]; then echo " IPv4 Address: ${ipv4_address}" echo " IPv4 Prefix Length: ${ipv4_prefix_length}" echo " IPv4 Gateway: ${ipv4_gateway}" - echo " Domain name : ${domain}" + fi + if [[ -n ${domain_name} ]]; then + echo " Domain name : ${domain_name}" fi echo " " @@ -126,10 +128,10 @@ function validate_domain_name() { local domain_part local -a domain_parts - IFS='.' read -ra domain_parts <<<"${domain}" + IFS='.' read -ra domain_parts <<<"${domain_name}" if [ ${#domain_parts[@]} -lt 2 ]; then - log_and_halt_installation_on_error 1 "Domain validation error: less than two domain parts in domain: ${domain}" + log_and_halt_installation_on_error 1 "Domain validation error: less than two domain parts in domain: ${domain_name}" fi for domain_part in "${domain_parts[@]}"; do From 74a6b2b708795a99e4643750a8a77abf3c9b835d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 4 Nov 2024 19:29:55 +0000 Subject: [PATCH 156/241] Add guestos log-config --- ic-os/components/guestos.bzl | 2 ++ ic-os/components/hostos.bzl | 4 ++-- .../{hostos-scripts => misc}/log-config/log-config.service | 2 +- .../{hostos-scripts => misc}/log-config/log-config.sh | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) rename ic-os/components/{hostos-scripts => misc}/log-config/log-config.service (75%) rename ic-os/components/{hostos-scripts => misc}/log-config/log-config.sh (94%) diff --git a/ic-os/components/guestos.bzl b/ic-os/components/guestos.bzl index 76d8022cde7..1926fc006c9 100644 --- a/ic-os/components/guestos.bzl +++ b/ic-os/components/guestos.bzl @@ -60,6 +60,8 @@ component_files = { Label("misc/guestos/sysctl.d/dfn-max-map-count.conf"): "/etc/sysctl.d/dfn-max-map-count.conf", Label("misc/guestos/sysctl.d/privileged-ports.conf"): "/etc/sysctl.d/privileged-ports.conf", Label("misc/guestos/sysfs.d/hugepage.conf"): "/etc/sysfs.d/hugepage.conf", + Label("misc/log-config/log-config.service"): "/etc/systemd/system/log-config.service", + Label("misc/log-config/log-config.sh"): "/opt/ic/bin/log-config.sh", # monitoring Label("monitoring/filebeat/setup-filebeat-permissions.sh"): "/opt/ic/bin/setup-filebeat-permissions.sh", diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index efbd147cfb8..e7a1dec346f 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -29,8 +29,6 @@ component_files = { Label("hostos-scripts/verbose-logging/verbose-logging.sh"): "/opt/ic/bin/verbose-logging.sh", Label("hostos-scripts/verbose-logging/verbose-logging.service"): "/etc/systemd/system/verbose-logging.service", Label("hostos-scripts/verbose-logging/logrotate.d/verbose-logging"): "/etc/logrotate.d/verbose-logging", - Label("hostos-scripts/log-config/log-config.service"): "/etc/systemd/system/log-config.service", - Label("hostos-scripts/log-config/log-config.sh"): "/opt/ic/bin/log-config.sh", # early-boot Label("early-boot/relabel-machine-id/relabel-machine-id.sh"): "/opt/ic/bin/relabel-machine-id.sh", @@ -59,6 +57,8 @@ component_files = { Label("misc/hostos/sudoers"): "/etc/sudoers", Label("misc/hostos/ic-node.conf"): "/etc/tmpfiles.d/ic-node.conf", Label("misc/hostos/20-ipmi.rules"): "/etc/udev/rules.d/20-ipmi.rules", + Label("misc/log-config/log-config.service"): "/etc/systemd/system/log-config.service", + Label("misc/log-config/log-config.sh"): "/opt/ic/bin/log-config.sh", # monitoring Label("monitoring/systemd-user/user@.service"): "/etc/systemd/system/user@.service", diff --git a/ic-os/components/hostos-scripts/log-config/log-config.service b/ic-os/components/misc/log-config/log-config.service similarity index 75% rename from ic-os/components/hostos-scripts/log-config/log-config.service rename to ic-os/components/misc/log-config/log-config.service index b5e319f5a6f..24f8ef30fc2 100644 --- a/ic-os/components/hostos-scripts/log-config/log-config.service +++ b/ic-os/components/misc/log-config/log-config.service @@ -1,5 +1,5 @@ [Unit] -Description=Log HostOS config partition +Description=Log config partition [Service] Type=oneshot diff --git a/ic-os/components/hostos-scripts/log-config/log-config.sh b/ic-os/components/misc/log-config/log-config.sh similarity index 94% rename from ic-os/components/hostos-scripts/log-config/log-config.sh rename to ic-os/components/misc/log-config/log-config.sh index bf311cde304..afb4655e306 100644 --- a/ic-os/components/hostos-scripts/log-config/log-config.sh +++ b/ic-os/components/misc/log-config/log-config.sh @@ -27,6 +27,6 @@ log_file_contents() { fi } -echo "Logging HostOS config partition" +echo "Logging config partition" log_directory_structure "$CONFIG_DIR" log_file_contents "$CONFIG" From a22722f47b9b6c74c0171f3b3bf8d26d93edbe69 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 4 Nov 2024 19:50:32 +0000 Subject: [PATCH 157/241] Depend log-config-guestos.service on bootstrap-ic-node.service --- ic-os/components/guestos.bzl | 2 +- .../misc/log-config/log-config-guestos.service | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 ic-os/components/misc/log-config/log-config-guestos.service diff --git a/ic-os/components/guestos.bzl b/ic-os/components/guestos.bzl index 1926fc006c9..eda3e00c6b2 100644 --- a/ic-os/components/guestos.bzl +++ b/ic-os/components/guestos.bzl @@ -60,7 +60,7 @@ component_files = { Label("misc/guestos/sysctl.d/dfn-max-map-count.conf"): "/etc/sysctl.d/dfn-max-map-count.conf", Label("misc/guestos/sysctl.d/privileged-ports.conf"): "/etc/sysctl.d/privileged-ports.conf", Label("misc/guestos/sysfs.d/hugepage.conf"): "/etc/sysfs.d/hugepage.conf", - Label("misc/log-config/log-config.service"): "/etc/systemd/system/log-config.service", + Label("misc/log-config/log-config-guestos.service"): "/etc/systemd/system/log-config.service", Label("misc/log-config/log-config.sh"): "/opt/ic/bin/log-config.sh", # monitoring diff --git a/ic-os/components/misc/log-config/log-config-guestos.service b/ic-os/components/misc/log-config/log-config-guestos.service new file mode 100644 index 00000000000..b5ee83f0bd0 --- /dev/null +++ b/ic-os/components/misc/log-config/log-config-guestos.service @@ -0,0 +1,12 @@ +[Unit] +Description=Log config partition +After=bootstrap-ic-node.service +Requires=bootstrap-ic-node.service + +[Service] +Type=oneshot +ExecStart=/opt/ic/bin/log-config.sh +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target \ No newline at end of file From af0af1f995992a44e665a35db1f422d9bcd031ea Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 5 Nov 2024 19:09:25 +0000 Subject: [PATCH 158/241] Move config_version to *OSConfig structs --- rs/ic_os/config/src/generate_testnet_config.rs | 2 +- rs/ic_os/config/src/lib.rs | 12 +++++++----- rs/ic_os/config/src/main.rs | 4 +++- rs/ic_os/config/src/types.rs | 8 ++++++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 6085e7945c1..c60504b543a 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -192,7 +192,6 @@ pub fn generate_testnet_config( let use_ssh_authorized_keys = use_ssh_authorized_keys.unwrap_or(true); let icos_settings = ICOSSettings { - config_version: CONFIG_VERSION.to_string(), mgmt_mac, deployment_environment, logging, @@ -241,6 +240,7 @@ pub fn generate_testnet_config( // Assemble GuestOSConfig let guestos_config = GuestOSConfig { + config_version: CONFIG_VERSION.to_string(), network_settings, icos_settings, guestos_settings, diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 249c15036ac..dc493c0d918 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -72,7 +72,6 @@ mod tests { }; let icos_dev_settings = ICOSDevSettings::default(); let icos_settings = ICOSSettings { - config_version: CONFIG_VERSION.to_string(), mgmt_mac: FormattedMacAddress::try_from("ec:2a:72:31:a2:0c")?, deployment_environment: "Mainnet".to_string(), logging, @@ -96,6 +95,7 @@ mod tests { }; let setupos_config_struct = SetupOSConfig { + config_version: CONFIG_VERSION.to_string(), network_settings: network_settings.clone(), icos_settings: icos_settings.clone(), setupos_settings: setupos_settings.clone(), @@ -103,12 +103,14 @@ mod tests { guestos_settings: guestos_settings.clone(), }; let hostos_config_struct = HostOSConfig { + config_version: CONFIG_VERSION.to_string(), network_settings: network_settings.clone(), icos_settings: icos_settings.clone(), hostos_settings: hostos_settings.clone(), guestos_settings: guestos_settings.clone(), }; let guestos_config_struct = GuestOSConfig { + config_version: CONFIG_VERSION.to_string(), network_settings: network_settings.clone(), icos_settings: icos_settings.clone(), guestos_settings: guestos_settings.clone(), @@ -141,6 +143,7 @@ mod tests { // Test config version 1.0.0 const HOSTOS_CONFIG_JSON_V1_0_0: &str = r#" { + "config_version": "1.0.0", "network_settings": { "ipv6_config": { "Deterministic": { @@ -157,7 +160,6 @@ mod tests { "domain_name": "example.com" }, "icos_settings": { - "config_version": "1.0.0", "mgmt_mac": "ec:2a:72:31:a2:0c", "deployment_environment": "Mainnet", "logging": { @@ -200,6 +202,7 @@ mod tests { const GUESTOS_CONFIG_JSON_V1_0_0: &str = r#" { + "config_version": "1.0.0", "network_settings": { "ipv6_config": { "Deterministic": { @@ -216,7 +219,6 @@ mod tests { "domain_name": "example.com" }, "icos_settings": { - "config_version": "1.0.0", "mgmt_mac": "ec:2a:72:31:a2:0c", "deployment_environment": "Mainnet", "logging": { @@ -255,7 +257,7 @@ mod tests { #[test] fn test_deserialize_hostos_config_v1_0_0() -> Result<(), Box> { let config: HostOSConfig = serde_json::from_str(HOSTOS_CONFIG_JSON_V1_0_0)?; - assert_eq!(config.icos_settings.config_version, "1.0.0"); + assert_eq!(config.config_version, "1.0.0"); assert_eq!(config.hostos_settings.vm_cpu, "kvm"); Ok(()) } @@ -263,7 +265,7 @@ mod tests { #[test] fn test_deserialize_guestos_config_v1_0_0() -> Result<(), Box> { let config: GuestOSConfig = serde_json::from_str(GUESTOS_CONFIG_JSON_V1_0_0)?; - assert_eq!(config.icos_settings.config_version, "1.0.0"); + assert_eq!(config.config_version, "1.0.0"); assert_eq!( config.icos_settings.mgmt_mac.to_string(), "ec:2a:72:31:a2:0c" diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index b81e534b197..4c2d34e0e27 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -207,7 +207,6 @@ pub fn main() -> Result<()> { }; let icos_settings = ICOSSettings { - config_version: CONFIG_VERSION.to_string(), mgmt_mac, deployment_environment: deployment_json_settings.deployment.name, logging, @@ -233,6 +232,7 @@ pub fn main() -> Result<()> { let guestos_settings = GuestOSSettings::default(); let setupos_config = SetupOSConfig { + config_version: CONFIG_VERSION.to_string(), network_settings, icos_settings, setupos_settings, @@ -261,6 +261,7 @@ pub fn main() -> Result<()> { serde_json::from_reader(File::open(setupos_config_json_path)?)?; let hostos_config = HostOSConfig { + config_version: setupos_config.config_version, network_settings: setupos_config.network_settings, icos_settings: setupos_config.icos_settings, hostos_settings: setupos_config.hostos_settings, @@ -306,6 +307,7 @@ pub fn main() -> Result<()> { } let guestos_config = GuestOSConfig { + config_version: hostos_config.config_version, network_settings: guestos_network_settings, icos_settings: hostos_config.icos_settings, guestos_settings: hostos_config.guestos_settings, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 65f918d8830..e0d18aa4eb4 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -26,6 +26,8 @@ pub const CONFIG_VERSION: &str = "1.0.0"; /// (e.g., `config.ini`, `deployment.json`) are transformed into `SetupOSConfig`. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct SetupOSConfig { + /// Tracks the config version, set to CONFIG_VERSION at runtime. + pub config_version: String, pub network_settings: NetworkSettings, pub icos_settings: ICOSSettings, pub setupos_settings: SetupOSSettings, @@ -36,6 +38,8 @@ pub struct SetupOSConfig { /// HostOS configuration. In production, this struct inherits settings from `SetupOSConfig`. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct HostOSConfig { + /// Tracks the config version, set to CONFIG_VERSION at runtime. + pub config_version: String, pub network_settings: NetworkSettings, pub icos_settings: ICOSSettings, pub hostos_settings: HostOSSettings, @@ -45,6 +49,8 @@ pub struct HostOSConfig { /// GuestOS configuration. In production, this struct inherits settings from `HostOSConfig`. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct GuestOSConfig { + /// Tracks the config version, set to CONFIG_VERSION at runtime. + pub config_version: String, pub network_settings: NetworkSettings, pub icos_settings: ICOSSettings, pub guestos_settings: GuestOSSettings, @@ -52,8 +58,6 @@ pub struct GuestOSConfig { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct ICOSSettings { - /// Tracks the config version, set to CONFIG_VERSION at runtime. - pub config_version: String, /// In nested testing, mgmt_mac is set in deployment.json.template, /// else found dynamically in call to config tool CreateSetuposConfig pub mgmt_mac: FormattedMacAddress, From b41564c5412e49f0b6f617e61916d077cb70f02b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 5 Nov 2024 23:05:07 +0000 Subject: [PATCH 159/241] Re-add deployment.json and config.ini copying to hostos config --- .../components/setupos-scripts/setup-hostos-config.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index efcc47dd8e6..0b2c556a505 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -21,6 +21,17 @@ function mount_config_partition() { } function copy_config_files() { + # TODO(NODE-1519): delete config.ini copying after switch to new icos config + echo "* Copying 'config.ini' to hostOS config partition..." + if [ -f "${CONFIG_DIR}/config.ini" ]; then + cp ${CONFIG_DIR}/config.ini /media/ + log_and_halt_installation_on_error "${?}" "Unable to copy 'config.ini' to hostOS config partition." + + # TODO(NODE-1519): delete deployment.json copying after switch to new icos config + echo "* Copying deployment.json to config partition..." + cp /data/deployment.json /media/ + log_and_halt_installation_on_error "${?}" "Unable to copy deployment.json to hostOS config partition." + echo "* Copying SSH authorized keys..." use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') if [[ "${use_ssh_authorized_keys,,}" == "true" ]]; then From 4ccc351f0dd77ec07d4324390e261136a36909c0 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 5 Nov 2024 19:26:34 +0000 Subject: [PATCH 160/241] Update config policy documentation --- rs/ic_os/config/README.md | 3 ++- rs/ic_os/config/src/types.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/rs/ic_os/config/README.md b/rs/ic_os/config/README.md index 61628086964..5be108377cc 100644 --- a/rs/ic_os/config/README.md +++ b/rs/ic_os/config/README.md @@ -1,6 +1,6 @@ # IC-OS Config -IC-OS Config is responsible for managing the configuration of IC-OS images. +IC-OS Config is responsible for managing the configuration of IC-OS images. SetupOS transforms user-facing configuration files (like `config.ini`, `deployment.json`, etc.) into a SetupOSConfig struct. Then, in production, configuration is propagated from SetupOS → HostOS → GuestOS (→ replica) via the HostOSConfig and GuestOSConfig structures. @@ -10,5 +10,6 @@ For testing, IC-OS Config is also used to create GuestOS configuration directly. When updating the IC-OS configuration, it's crucial to ensure backwards compatibility. For detailed guidelines on updating the configuration, please refer to the documentation in [`types.rs`](src/types.rs). +Any changes to the configuration should undergo a thorough review process to ensure they follow the guidlines. For details on the IC-OS configuration mechanism, refer to [ic-os/docs/Configuration.adoc](../../../ic-os/docs/Configuration.adoc) \ No newline at end of file diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index e0d18aa4eb4..68dda00eb34 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -11,7 +11,9 @@ //! //! - **Adding New Fields**: If adding a new field to a configuration struct, make sure it is optional or has a default value by implementing `Default` or via `#[serde(default)]`. //! -//! - **Removing Fields**: If removing a field, ensure all references to it in the IC-OS codebase are eliminated. +//! - **Removing Fields**: To prevent backwards-compatibility deserialization errors, required fields must not be removed directly: +//! In a first step, they have to be made optional and code that reads the value must be removed/handle missing values. +//! In a second step, after the first step has rolled out to all OSes and there is no risk of a rollback, the field can be removed. //! //! - **Renaming Fields**: Avoid renaming fields unless absolutely necessary. If you must rename a field, use `#[serde(rename = "old_name")]`. use ic_types::malicious_behaviour::MaliciousBehaviour; From 24469aec072eb06ad0c87cbed9c97972cd583e6e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 5 Nov 2024 23:01:18 +0000 Subject: [PATCH 161/241] Fix cargo clippy --- rs/ic_os/config/src/types.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 68dda00eb34..50a53e17fbb 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -11,9 +11,7 @@ //! //! - **Adding New Fields**: If adding a new field to a configuration struct, make sure it is optional or has a default value by implementing `Default` or via `#[serde(default)]`. //! -//! - **Removing Fields**: To prevent backwards-compatibility deserialization errors, required fields must not be removed directly: -//! In a first step, they have to be made optional and code that reads the value must be removed/handle missing values. -//! In a second step, after the first step has rolled out to all OSes and there is no risk of a rollback, the field can be removed. +//! - **Removing Fields**: To prevent backwards-compatibility deserialization errors, required fields must not be removed directly: In a first step, they have to be made optional and code that reads the value must be removed/handle missing values. In a second step, after the first step has rolled out to all OSes and there is no risk of a rollback, the field can be removed. //! //! - **Renaming Fields**: Avoid renaming fields unless absolutely necessary. If you must rename a field, use `#[serde(rename = "old_name")]`. use ic_types::malicious_behaviour::MaliciousBehaviour; From 9af8e049fe0f24f1b5c2ca151b9953f357a888c3 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 6 Nov 2024 16:06:49 +0000 Subject: [PATCH 162/241] Merge branch 'master' into andrew/config-revamp-integration --- Cargo.lock | 84 +++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ce309f4e45..5d999be5d07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2574,18 +2574,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5e7afe85cadb55c4c1176268a2ac046fdff8dfaeca39e18581b9dc319ca9e" +checksum = "540b193ff98b825a1f250a75b3118911af918a734154c69d80bcfcf91e7e9522" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ab25ef3be935a80680e393183e1f94ef507e93a24a8369494d2c6818aedb3e3" +checksum = "c7cb269598b9557ab942d687d3c1086d77c4b50dcf35813f3a65ba306fd42279" dependencies = [ "serde", "serde_derive", @@ -2593,9 +2593,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900a19b84545924f1851cbfe386962edfc4ecbc3366a254825cf1ecbcda8ba08" +checksum = "46566d7c83a8bff4150748d66020f4c7224091952aa4b4df1ec4959c39d937a1" dependencies = [ "bumpalo", "cranelift-bforest", @@ -2616,33 +2616,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c73b2395ffe9e7b4fdf7e2ebc052e7e27af13f68a964985346be4da477a5fc" +checksum = "2df8a86a34236cc75a8a6a271973da779c2aeb36c43b6e14da474cf931317082" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d9ed0854e96a4ff0879bff39d078de8dea7f002721c9494c1fdb4e1baa86ccc" +checksum = "cf75340b6a57b7c7c1b74f10d3d90883ee6d43a554be8131a4046c2ebcf5eb65" [[package]] name = "cranelift-control" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4aca921dd422e781409de0129c255768fec5dec1dae83239b497fb9138abb89" +checksum = "2e84495bc5d23d86aad8c86f8ade4af765b94882af60d60e271d3153942f1978" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d770e6605eccee15b49decdd82cd26f2b6404767802471459ea49c57379a98" +checksum = "963c17147b80df351965e57c04d20dbedc85bcaf44c3436780a59a3f1ff1b1c2" dependencies = [ "cranelift-bitset", "serde", @@ -2651,9 +2651,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29268711cb889cb39215b10faf88b9087d4c9e1d2633581e4f722a2bf4bb4ef9" +checksum = "727f02acbc4b4cb2ba38a6637101d579db50190df1dd05168c68e762851a3dd5" dependencies = [ "cranelift-codegen", "log", @@ -2663,15 +2663,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc65156f010aed1985767ad1bff0eb8d186743b7b03e23d0c17604a253e3f356" +checksum = "32b00cc2e03c748f2531eea01c871f502b909d30295fdcad43aec7bf5c5b4667" [[package]] name = "cranelift-native" -version = "0.113.0" +version = "0.113.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8bf9b361eaf5a7627647270fabf1dc910d993edbeaf272a652c107861ebe9c2" +checksum = "bbeaf978dc7c1a2de8bbb9162510ed218eb156697bc45590b8fbdd69bb08e8de" dependencies = [ "cranelift-codegen", "libc", @@ -17475,9 +17475,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68c610ff29655a42eeef41a5b5346e714586971a7d927739477e552fe7e23e3" +checksum = "df33e7f8a43ccc7f93b330fef4baf271764674926f3f4d40f4a196d54de8af26" dependencies = [ "cranelift-bitset", "log", @@ -21758,9 +21758,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffa3230b9ba1ab6568d116df21bf4ca55ed2bfac87723d910471d30d9656ea1" +checksum = "51e762e163fd305770c6c341df3290f0cabb3c264e7952943018e9a1ced8d917" dependencies = [ "anyhow", "bitflags 2.6.0", @@ -21800,18 +21800,18 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef15fad08bbaa0e5c5539b76fa5965ca25e24f17a584f83a40b43ba9a2b36f44" +checksum = "63caa7aebb546374e26257a1900fb93579171e7c02514cde26805b9ece3ef812" dependencies = [ "cfg-if 1.0.0", ] [[package]] name = "wasmtime-component-macro" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb4e179f424260d0739c09d3bc83d34347a55d291d10dcb5244686a75c7733" +checksum = "d61a4b5ce2ad9c15655e830f0eac0c38b8def30c74ecac71f452d3901e491b68" dependencies = [ "anyhow", "proc-macro2", @@ -21824,15 +21824,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe3c27d64af5f584014db9381c081223d27a57e1dce2f6280bbafea37575619" +checksum = "35e87a1212270dbb84a49af13d82594e00a92769d6952b0ea7fc4366c949f6ad" [[package]] name = "wasmtime-cranelift" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb56d9ee4a093509624bd0861888cd111f6530e16969a68bb12dc7dd7a2be27f" +checksum = "7cb40dddf38c6a5eefd5ce7c1baf43b00fe44eada11a319fab22e993a960262f" dependencies = [ "anyhow", "cfg-if 1.0.0", @@ -21855,9 +21855,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3444c1759d5b906ff76a3cab073dd92135bdd06e5d1f46635ec40a58207d314" +checksum = "8613075e89e94a48c05862243c2b718eef1b9c337f51493ebf951e149a10fa19" dependencies = [ "anyhow", "cranelift-bitset", @@ -21878,9 +21878,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e458e6a1a010a53f86ac8d75837c0c6b2ce3e54b7503b2f1dc5629a4a541f5a" +checksum = "da47fba49af72581bc0dc67c8faaf5ee550e6f106e285122a184a675193701a5" dependencies = [ "anyhow", "cfg-if 1.0.0", @@ -21890,15 +21890,15 @@ dependencies = [ [[package]] name = "wasmtime-slab" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339c9a2a62b989a3184baff31be3a5b5256ad52629634eb432f9ccf0ab251f83" +checksum = "770e10cdefb15f2b6304152978e115bd062753c1ebe7221c0b6b104fa0419ff6" [[package]] name = "wasmtime-versioned-export-macros" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe01058e422966659e1af00af833147d54658b07c7e74606d73ca9af3f1690a" +checksum = "db8efb877c9e5e67239d4553bb44dd2a34ae5cfb728f3cf2c5e64439c6ca6ee7" dependencies = [ "proc-macro2", "quote", @@ -21907,9 +21907,9 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "26.0.0" +version = "26.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c9e85935a1199e96b73e7fcd27a127035d2082265720a67d59268a24892d567" +checksum = "4bef2a726fd8d1ee9b0144655e16c492dc32eb4c7c9f7e3309fcffe637870933" dependencies = [ "anyhow", "heck 0.5.0", From 72e2e8ea0f12d7d0de0671ef0acb27d24f4c3cbd Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 6 Nov 2024 16:25:57 +0000 Subject: [PATCH 163/241] Fix copy_config_files --- ic-os/components/setupos-scripts/setup-hostos-config.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 0b2c556a505..b3c5e1bc7ac 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -26,7 +26,10 @@ function copy_config_files() { if [ -f "${CONFIG_DIR}/config.ini" ]; then cp ${CONFIG_DIR}/config.ini /media/ log_and_halt_installation_on_error "${?}" "Unable to copy 'config.ini' to hostOS config partition." - + else + log_and_halt_installation_on_error "1" "Configuration file 'config.ini' does not exist." + fi + # TODO(NODE-1519): delete deployment.json copying after switch to new icos config echo "* Copying deployment.json to config partition..." cp /data/deployment.json /media/ From 87ebfaf6b5ab5c67da247fdb0307fc3e1887c316 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 12 Nov 2024 16:29:21 +0000 Subject: [PATCH 164/241] Fix generate-ic-config.sh --- .../generate-ic-config.service | 2 +- .../generate-ic-config/generate-ic-config.sh | 244 ++++++------------ 2 files changed, 85 insertions(+), 161 deletions(-) diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.service b/ic-os/components/ic/generate-ic-config/generate-ic-config.service index 3d6d08311f3..ea5efe29a4b 100644 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.service +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.service @@ -16,7 +16,7 @@ Wants=network-online.target [Service] Type=oneshot -ExecStart=/opt/ic/bin/generate-ic-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -t /boot/config/jaeger_addr.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 +ExecStart=/opt/ic/bin/generate-ic-config.sh -n -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 [Install] WantedBy=multi-user.target diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh index b19d666e475..12630886578 100755 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh @@ -3,24 +3,73 @@ # Substitute correct configuration parameters into ic.json5. Will take IP addresses # from configuration file or from network interfaces. +source /opt/ic/bin/config.sh + function usage() { cat < Date: Tue, 12 Nov 2024 17:39:18 +0000 Subject: [PATCH 165/241] Fix generate-ic-config.service --- .../components/ic/generate-ic-config/generate-ic-config.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.service b/ic-os/components/ic/generate-ic-config/generate-ic-config.service index ea5efe29a4b..72c618efa97 100644 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.service +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.service @@ -16,7 +16,7 @@ Wants=network-online.target [Service] Type=oneshot -ExecStart=/opt/ic/bin/generate-ic-config.sh -n -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 +ExecStart=/opt/ic/bin/generate-ic-config.sh -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 [Install] WantedBy=multi-user.target From ece61d6c8e4a41c6682aa5773eab95b542cf98e1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 12 Nov 2024 19:27:44 +0000 Subject: [PATCH 166/241] Remove config.sh reference --- ic-os/components/setupos.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/ic-os/components/setupos.bzl b/ic-os/components/setupos.bzl index 5b930002812..b60e576cd8c 100644 --- a/ic-os/components/setupos.bzl +++ b/ic-os/components/setupos.bzl @@ -9,7 +9,6 @@ component_files = { # setupos-scripts Label("//ic-os/components/setupos-scripts:check-setupos-age.sh"): "/opt/ic/bin/check-setupos-age.sh", Label("//ic-os/components/setupos-scripts:check-config.sh"): "/opt/ic/bin/check-config.sh", - Label("//ic-os/components/setupos-scripts:config.sh"): "/opt/ic/bin/config.sh", Label("//ic-os/components/setupos-scripts:setup-hostos-config.sh"): "/opt/ic/bin/setup-hostos-config.sh", Label("//ic-os/components/setupos-scripts:setup-disk.sh"): "/opt/ic/bin/setup-disk.sh", Label("//ic-os/components/setupos-scripts:functions.sh"): "/opt/ic/bin/functions.sh", From 13c7f56ecdf47636a511988c97693a433b4c2e08 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 15 Nov 2024 21:37:30 +0000 Subject: [PATCH 167/241] Add node_reward_type to config tool --- rs/ic_os/config/src/config_ini.rs | 4 ++++ rs/ic_os/config/src/generate_testnet_config.rs | 5 +++++ rs/ic_os/config/src/lib.rs | 3 +++ rs/ic_os/config/src/main.rs | 7 +++++++ rs/ic_os/config/src/types.rs | 3 +++ 5 files changed, 22 insertions(+) diff --git a/rs/ic_os/config/src/config_ini.rs b/rs/ic_os/config/src/config_ini.rs index c37b8f9d178..cbea0a5bee6 100644 --- a/rs/ic_os/config/src/config_ini.rs +++ b/rs/ic_os/config/src/config_ini.rs @@ -17,6 +17,7 @@ pub struct ConfigIniSettings { pub ipv4_prefix_length: Option, pub domain_name: Option, pub verbose: bool, + pub node_reward_type: Option, } // Prefix should have a max length of 19 ("1234:6789:1234:6789") @@ -88,6 +89,8 @@ pub fn get_config_ini_settings(config_file_path: &Path) -> Result Result, // ICOSSettings arguments + pub node_reward_type: Option, pub mgmt_mac: Option, pub deployment_environment: Option, pub elasticsearch_hosts: Option, @@ -69,6 +70,7 @@ fn create_guestos_config(config: GenerateTestnetConfigArgs) -> Result Result FormattedMacAddress::try_from(mac_str.as_str())?, None => { @@ -189,6 +193,7 @@ fn create_guestos_config(config: GenerateTestnetConfigArgs) -> Result, + #[arg(long)] pub mgmt_mac: Option, #[arg(long)] pub deployment_environment: Option, @@ -158,6 +160,7 @@ pub fn main() -> Result<()> { ipv4_prefix_length, domain_name, verbose, + node_reward_type, } = get_config_ini_settings(&config_ini_path)?; // create NetworkSettings @@ -206,7 +209,10 @@ pub fn main() -> Result<()> { None => get_ipmi_mac()?, }; + let node_reward_type = node_reward_type.expect("Node reward type is required."); + let icos_settings = ICOSSettings { + node_reward_type, mgmt_mac, deployment_environment: deployment_json_settings.deployment.name, logging, @@ -336,6 +342,7 @@ pub fn main() -> Result<()> { ipv4_gateway: clap_args.ipv4_gateway, ipv4_prefix_length: clap_args.ipv4_prefix_length, domain_name: clap_args.domain_name, + node_reward_type: clap_args.node_reward_type, mgmt_mac: clap_args.mgmt_mac, deployment_environment: clap_args.deployment_environment, elasticsearch_hosts: clap_args.elasticsearch_hosts, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index f1dce16697b..2b3fe8417bb 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -61,6 +61,8 @@ pub struct GuestOSConfig { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct ICOSSettings { + /// The node reward type determines node rewards + pub node_reward_type: String, /// In nested testing, mgmt_mac is set in deployment.json.template, /// else found dynamically in call to config tool CreateSetuposConfig pub mgmt_mac: FormattedMacAddress, @@ -200,6 +202,7 @@ mod tests { domain_name: None, }, icos_settings: ICOSSettings { + node_reward_type: String::new(), mgmt_mac: FormattedMacAddress::try_from("00:00:00:00:00:00")?, deployment_environment: String::new(), logging: Logging { From c744a6c67a5474be1d181cc9b089d52d2dfb75e7 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 15 Nov 2024 21:40:54 +0000 Subject: [PATCH 168/241] Add node_reward_type regex check --- rs/ic_os/config/src/main.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 0a4c22dc3f9..b6872f5de1b 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -4,6 +4,7 @@ use config::config_ini::{get_config_ini_settings, ConfigIniSettings}; use config::deployment_json::get_deployment_settings; use config::serialize_and_write_config; use mac_address::mac_address::{get_ipmi_mac, FormattedMacAddress}; +use regex::Regex; use std::fs::File; use std::path::{Path, PathBuf}; @@ -211,6 +212,14 @@ pub fn main() -> Result<()> { let node_reward_type = node_reward_type.expect("Node reward type is required."); + let node_reward_type_pattern = Regex::new(r"^type[0-9]+(\.[0-9])?$")?; + if !node_reward_type_pattern.is_match(&node_reward_type) { + anyhow::bail!( + "Invalid node_reward_type '{}'. It must match the pattern ^type[0-9]+(\\.[0-9])?$", + node_reward_type + ); + } + let icos_settings = ICOSSettings { node_reward_type, mgmt_mac, From afc6da031c5242b0f34b15b3890f458f6d38d95d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 15 Nov 2024 21:48:35 +0000 Subject: [PATCH 169/241] Remove validate_node_reward --- .../setupos-scripts/check-hardware.sh | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/ic-os/components/setupos-scripts/check-hardware.sh b/ic-os/components/setupos-scripts/check-hardware.sh index 31c6e7c2f40..acf66ab5861 100644 --- a/ic-os/components/setupos-scripts/check-hardware.sh +++ b/ic-os/components/setupos-scripts/check-hardware.sh @@ -261,32 +261,6 @@ function verify_deployment_path() { fi } -# TODO(NODE-1477): delete in configuration revamp integration -CONFIG="${CONFIG:=/var/ic/config/config.ini}" - -function read_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "node_reward_type") node_reward_type="${value}" ;; - esac - done <"${CONFIG}" -} - -function validate_node_reward() { - read_variables - if [[ -z "$node_reward_type" ]]; then - log_and_halt_installation_on_error 1 "Configuration error: node_reward_type is not set" - fi - - if [[ ! "$node_reward_type" =~ ^type[0-9]+(\.[0-9])?$ ]]; then - log_and_halt_installation_on_error 1 "Configuration error: node_reward_type is invalid: ${node_reward_type}" - fi - - echo "Valid node reward type: ${node_reward_type}" -} - # Establish run order main() { log_start "$(basename $0)" @@ -296,7 +270,6 @@ main() { verify_memory verify_disks verify_deployment_path - validate_node_reward else echo "* Hardware checks skipped by request via kernel command line" GENERATION=2 From f876bce0fc82d8dc4d69f2706cba59b59e7c01f2 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 15 Nov 2024 21:50:21 +0000 Subject: [PATCH 170/241] Remove node_reward_type from static testnets --- testnet/tools/build-guestos-configs.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/testnet/tools/build-guestos-configs.sh b/testnet/tools/build-guestos-configs.sh index a430bfae790..9c68a291114 100755 --- a/testnet/tools/build-guestos-configs.sh +++ b/testnet/tools/build-guestos-configs.sh @@ -320,7 +320,6 @@ function build_bootstrap_images() { ${use_crypto:+"--ic_crypto"} ${use_crypto:+"${IC_PREP_DIR}/node-${node_idx}/crypto/"} \ "--nns_urls" "${NNS_URLS}" \ "--nns_public_key" "${IC_PREP_DIR}/nns_public_key.pem" \ - "--node_reward_type type3.1" \ "--hostname" "${hostname}" \ "--accounts_ssh_authorized_keys" "${SSH}" \ ${ELASTICSEARCH_HOSTS:+"--elasticsearch_hosts"} ${ELASTICSEARCH_HOSTS:+"${ELASTICSEARCH_HOSTS}"} \ From 5c1d00a7a837e6975cb212b708c25b5e09f46f74 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 15 Nov 2024 22:09:18 +0000 Subject: [PATCH 171/241] Make node_reward_type an optional --- rs/ic_os/config/src/generate_testnet_config.rs | 2 +- rs/ic_os/config/src/lib.rs | 2 +- rs/ic_os/config/src/main.rs | 2 +- rs/ic_os/config/src/types.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index 119f280f6d4..c619ccd355e 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -193,7 +193,7 @@ fn create_guestos_config(config: GenerateTestnetConfigArgs) -> Result Result<()> { } let icos_settings = ICOSSettings { - node_reward_type, + node_reward_type: Some(node_reward_type), mgmt_mac, deployment_environment: deployment_json_settings.deployment.name, logging, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 2b3fe8417bb..165ebdedc3c 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -62,7 +62,7 @@ pub struct GuestOSConfig { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct ICOSSettings { /// The node reward type determines node rewards - pub node_reward_type: String, + pub node_reward_type: Option, /// In nested testing, mgmt_mac is set in deployment.json.template, /// else found dynamically in call to config tool CreateSetuposConfig pub mgmt_mac: FormattedMacAddress, @@ -202,7 +202,7 @@ mod tests { domain_name: None, }, icos_settings: ICOSSettings { - node_reward_type: String::new(), + node_reward_type: Some(String::new()), mgmt_mac: FormattedMacAddress::try_from("00:00:00:00:00:00")?, deployment_environment: String::new(), logging: Logging { From 8e2cf634355e2577dee521d7643bdf9313bda739 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 15 Nov 2024 22:13:56 +0000 Subject: [PATCH 172/241] Add node_reward_type to farm and launch-single-vm config constructs --- rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs | 1 + rs/tests/driver/src/driver/bootstrap.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index d468a38e882..a238f14a2bc 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -216,6 +216,7 @@ fn main() { ipv4_address: None, ipv4_gateway: None, ipv4_prefix_length: None, + node_reward_type: None, domain_name: None, mgmt_mac: None, deployment_environment: Some("testnet".to_string()), diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index ec049b00630..0302fd18a85 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -435,6 +435,7 @@ fn create_config_disk_image( ipv4_gateway: None, ipv4_prefix_length: None, domain_name: None, + node_reward_type: None, mgmt_mac: None, deployment_environment: Some("testnet".to_string()), elasticsearch_hosts: None, From 7ab4f3080c53296c11cb8526f2c287ea3bd9d33e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 19 Nov 2024 21:15:14 +0000 Subject: [PATCH 173/241] Correct reference to NODE-1519 --- ic-os/components/setupos-scripts/setup-hostos-config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index 59d21d51f00..c2aa501062b 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -21,7 +21,7 @@ function mount_config_partition() { } function copy_config_files() { - # TODO(NODE-1519): delete config.ini copying after switch to new icos config + # TODO(NODE-1518): delete config.ini copying after switch to new icos config echo "* Copying 'config.ini' to hostOS config partition..." if [ -f "${CONFIG_DIR}/config.ini" ]; then cp ${CONFIG_DIR}/config.ini /media/ @@ -30,7 +30,7 @@ function copy_config_files() { log_and_halt_installation_on_error "1" "Configuration file 'config.ini' does not exist." fi - # TODO(NODE-1519): delete deployment.json copying after switch to new icos config + # TODO(NODE-1518): delete deployment.json copying after switch to new icos config echo "* Copying deployment.json to config partition..." cp /data/deployment.json /media/ log_and_halt_installation_on_error "${?}" "Unable to copy deployment.json to hostOS config partition." From 4edc6225507d9575b38b18f92a40fc2205d0358d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 19 Nov 2024 22:11:07 +0000 Subject: [PATCH 174/241] Pass old hostOS config to guestos --- .../check_file_references.py | 7 +- .../build-bootstrap-config-image.sh | 178 ++++++++++++++++++ .../dev-generate-guestos-config.sh | 42 +++++ .../generate-guestos-config.sh | 42 +++++ ic-os/components/hostos.bzl | 1 + .../bootstrap-ic-node/bootstrap-ic-node.sh | 3 +- ic-os/components/misc/fetch-property.sh | 97 ++++++++++ 7 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 ic-os/components/misc/fetch-property.sh diff --git a/ic-os/components/conformance_tests/check_file_references.py b/ic-os/components/conformance_tests/check_file_references.py index cf5425fe3ca..29947eb2e5d 100755 --- a/ic-os/components/conformance_tests/check_file_references.py +++ b/ic-os/components/conformance_tests/check_file_references.py @@ -11,7 +11,12 @@ import tarfile import tempfile -ALLOWED_UNDECLARED_DEPENDENCIES = {} +ALLOWED_UNDECLARED_DEPENDENCIES = { + "ic-os/components/misc/fetch-property.sh": { + # fetch-property.sh checks existence of metrics.sh + "/opt/ic/bin/metrics.sh", + } +} # Check file patterns /opt/ic/... COMPONENT_FILE_PATTERN = r"/opt/ic/[^\s'\"},)]+" diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 93cbe90597b..957ce14b8ab 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -45,6 +45,76 @@ options may be specified: --node_operator_private_key path Should point to a file containing a Node Provider private key PEM. + + --ipv6_address a:b::c/n + The IPv6 address to assign. Must include netmask in bits (e.g. + dead:beef::1/64). Overrides all other generation for testing. + + --ipv6_gateway a:b::c + Default IPv6 gateway. + + --ipv4_address a.b.c.d/n + (optional) The IPv4 address to assign. Must include prefix length (e.g. + 18.208.190.35/28). + + --ipv4_gateway a.b.c.d + (optional) Default IPv4 gateway (e.g. 18.208.190.33). + + --domain domain + (optional) The domain name to assign to the guest. + + --node_reward_type node_reward_type + (optional) The node reward type determines node rewards + + --hostname name + Name to assign to the host. Will be used in logging. + + --elasticsearch_hosts hosts + Logging hosts to use. Can be multiple hosts separated by space (make sure + to quote the argument string so it appears as a single argument to the + script, e.g. --elasticsearch_hosts "h1.domain.tld h2.domain.tld"). + + --elasticsearch_tags tags + Tags to be used by Filebeat. Can be multiple tags separated by space + (make sure to quote the argument string so it appears as a single argument + to the script, e.g. --elasticsearch_tags "testnet1 slo") + + --nns_urls urls + URL of NNS nodes for sign up or registry access. Can be multiple nodes + separated by commas. + + --backup_retention_time seconds + How long the backed up consensus artifacts should stay on the spool + before they get purged. + + --backup_puging_interval seconds + How often the backup purging should be executed. + + --malicious_behavior malicious_behavior + A JSON-object that describes the malicious behavior activated on + the node. This is only used for testing. + The Json-object corresponds to this Rust-structure: + ic_types::malicious_behaviour::MaliciousBehaviour + + --query_stats_epoch_length length + The length of the epoch in seconds. To be used in + systems tests only. + + --bitcoind_addr address + The IP address of a running bitcoind instance. To be used in + systems tests only. + + --jaeger_addr address + The IP address of a running Jaeger Collector instance. To be used in + systems tests only. + + --socks_proxy url + The URL of the socks proxy to use. To be used in + systems tests only. + + --generate_ic_boundary_tls_cert domain_name + Generate and inject a self-signed TLS certificate and key for ic-boundary + for the given domain name. To be used in system tests only. EOF } @@ -59,6 +129,15 @@ function build_ic_bootstrap_tar() { local NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY ACCOUNTS_SSH_AUTHORIZED_KEYS local IC_CRYPTO IC_STATE IC_REGISTRY_LOCAL_STORE + local IPV6_ADDRESS IPV6_GATEWAY DOMAIN HOSTNAME + local NNS_URLS + local BACKUP_RETENTION_TIME_SECS BACKUP_PURGING_INTERVAL_SECS + local ELASTICSEARCH_HOSTS ELASTICSEARCH_TAGS + local MALICIOUS_BEHAVIOR + local QUERY_STATS_EPOCH_LENGTH + local BITCOIND_ADDR + local JAEGER_ADDR + while true; do if [ $# == 0 ]; then break @@ -86,6 +165,60 @@ function build_ic_bootstrap_tar() { --ic_registry_local_store) IC_REGISTRY_LOCAL_STORE="$2" ;; + --ipv6_address) + IPV6_ADDRESS="$2" + ;; + --ipv6_gateway) + IPV6_GATEWAY="$2" + ;; + --ipv4_address) + IPV4_ADDRESS="$2" + ;; + --ipv4_gateway) + IPV4_GATEWAY="$2" + ;; + --domain) + DOMAIN="$2" + ;; + --node_reward_type) + NODE_REWARD_TYPE="$2" + ;; + --hostname) + HOSTNAME="$2" + ;; + --elasticsearch_hosts) + ELASTICSEARCH_HOSTS="$2" + ;; + --elasticsearch_tags) + ELASTICSEARCH_TAGS="$2" + ;; + --nns_urls) + NNS_URLS="$2" + ;; + --backup_retention_time) + BACKUP_RETENTION_TIME_SECS="$2" + ;; + --backup_puging_interval) + BACKUP_PURGING_INTERVAL_SECS="$2" + ;; + --malicious_behavior) + MALICIOUS_BEHAVIOR="$2" + ;; + --query_stats_epoch_length) + QUERY_STATS_EPOCH_LENGTH="$2" + ;; + --bitcoind_addr) + BITCOIND_ADDR="$2" + ;; + --jaeger_addr) + JAEGER_ADDR="$2" + ;; + --socks_proxy) + SOCKS_PROXY="$2" + ;; + --generate_ic_boundary_tls_cert) + IC_BOUNDARY_TLS_CERT_DOMAIN_NAME="$2" + ;; *) echo "Unrecognized option: $1" usage @@ -122,6 +255,51 @@ function build_ic_bootstrap_tar() { cp -r "${IC_REGISTRY_LOCAL_STORE}" "${BOOTSTRAP_TMPDIR}/ic_registry_local_store" fi + # TODO(NODE-1518): remove parsing for old config + [[ "$HOSTNAME" == "" ]] || [[ "$HOSTNAME" =~ [a-zA-Z]*([a-zA-Z0-9])*(-+([a-zA-Z0-9])) ]] || { + echo "Invalid hostname: '$HOSTNAME'" >&2 + exit 1 + } + + cat >"${BOOTSTRAP_TMPDIR}/network.conf" <"${BOOTSTRAP_TMPDIR}/reward.conf" + fi + if [ "${ELASTICSEARCH_HOSTS}" != "" ]; then + echo "elasticsearch_hosts=$ELASTICSEARCH_HOSTS" >"${BOOTSTRAP_TMPDIR}/filebeat.conf" + fi + if [ "${ELASTICSEARCH_TAGS}" != "" ]; then + echo "elasticsearch_tags=$ELASTICSEARCH_TAGS" >>"${BOOTSTRAP_TMPDIR}/filebeat.conf" + + if [ "${NNS_URLS}" != "" ]; then + echo "nns_url=${NNS_URLS}" >"${BOOTSTRAP_TMPDIR}/nns.conf" + fi + if [ "${BACKUP_RETENTION_TIME_SECS}" != "" ] || [ "${BACKUP_PURGING_INTERVAL_SECS}" != "" ]; then + echo "backup_retention_time_secs=${BACKUP_RETENTION_TIME_SECS}" >"${BOOTSTRAP_TMPDIR}/backup.conf" + echo "backup_puging_interval_secs=${BACKUP_PURGING_INTERVAL_SECS}" >>"${BOOTSTRAP_TMPDIR}/backup.conf" + fi + if [ "${MALICIOUS_BEHAVIOR}" != "" ]; then + echo "malicious_behavior=${MALICIOUS_BEHAVIOR}" >"${BOOTSTRAP_TMPDIR}/malicious_behavior.conf" + fi + if [ "${QUERY_STATS_EPOCH_LENGTH}" != "" ]; then + echo "query_stats_epoch_length=${QUERY_STATS_EPOCH_LENGTH}" >"${BOOTSTRAP_TMPDIR}/query_stats.conf" + fi + if [ "${BITCOIND_ADDR}" != "" ]; then + echo "bitcoind_addr=${BITCOIND_ADDR}" >"${BOOTSTRAP_TMPDIR}/bitcoind_addr.conf" + fi + if [ "${JAEGER_ADDR}" != "" ]; then + echo "jaeger_addr=http://${JAEGER_ADDR}" >"${BOOTSTRAP_TMPDIR}/jaeger_addr.conf" + + if [ "${SOCKS_PROXY}" != "" ]; then + echo "socks_proxy=${SOCKS_PROXY}" >"${BOOTSTRAP_TMPDIR}/socks_proxy.conf" + tar cf "${OUT_FILE}" \ --sort=name \ --owner=root:0 \ diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 08c377c805a..fe4b702ac92 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -11,11 +11,21 @@ source /opt/ic/bin/config.sh # Get keyword arguments for argument in "${@}"; do case ${argument} in + -c=* | --config=*) + CONFIG="${argument#*=}" + shift + ;; + -d=* | --deployment=*) + DEPLOYMENT="${argument#*=}" + shift + ;; -h | --help) echo 'Usage: Generate GuestOS Configuration Arguments: + -c=, --config= specify the config.ini configuration file (Default: /boot/config/config.ini) + -d=, --deployment= specify the deployment.json configuration file (Default: /boot/config/deployment.json) -h, --help show this help message and exit -i=, --input= specify the input template file (Default: /opt/ic/share/guestos.xml.template) -m=, --media= specify the config media image file (Default: /run/ic-node/config.img) @@ -49,10 +59,28 @@ function validate_arguments() { } # Set arguments if undefined +CONFIG="${CONFIG:=/boot/config/config.ini}" +DEPLOYMENT="${DEPLOYMENT:=/boot/config/deployment.json}" INPUT="${INPUT:=/opt/ic/share/guestos.xml.template}" MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" +# TODO(NODE-1518): remove passing old config +function read_old_config_variables() { + # Read limited set of keys. Be extra-careful quoting values as it could + # otherwise lead to executing arbitrary shell code! + while IFS="=" read -r key value; do + case "$key" in + "ipv6_prefix") ipv6_prefix="${value}" ;; + "ipv6_gateway") ipv6_gateway="${value}" ;; + "ipv4_address") ipv4_address="${value}" ;; + "ipv4_prefix_length") ipv4_prefix_length="${value}" ;; + "ipv4_gateway") ipv4_gateway="${value}" ;; + "domain") domain="${value}" ;; + "node_reward_type") node_reward_type="${value}" ;; + esac + done <"${CONFIG}" + function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') @@ -80,6 +108,19 @@ function assemble_config_media() { cmd+=(--accounts_ssh_authorized_keys "/boot/config/ssh_authorized_keys") fi + cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") + cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") + cmd+=(--ipv6_gateway "${ipv6_gateway}") + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then + cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") + cmd+=(--ipv4_gateway "${ipv4_gateway}") + cmd+=(--domain "${domain}") + fi + if [[ -n "$node_reward_type" ]]; then + cmd+=(--node_reward_type "${node_reward_type}") + cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") + # Run the above command "${cmd[@]}" write_log "Assembling config media for GuestOS: ${MEDIA}" @@ -119,6 +160,7 @@ function generate_guestos_config() { function main() { validate_arguments + read_old_config_variables read_config_variables assemble_config_media generate_guestos_config diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index bdd682b22fd..859c8978c98 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -11,11 +11,21 @@ source /opt/ic/bin/config.sh # Get keyword arguments for argument in "${@}"; do case ${argument} in + -c=* | --config=*) + CONFIG="${argument#*=}" + shift + ;; + -d=* | --deployment=*) + DEPLOYMENT="${argument#*=}" + shift + ;; -h | --help) echo 'Usage: Generate GuestOS Configuration Arguments: + -c=, --config= specify the config.ini configuration file (Default: /boot/config/config.ini) + -d=, --deployment= specify the deployment.json configuration file (Default: /boot/config/deployment.json) -h, --help show this help message and exit -i=, --input= specify the input template file (Default: /opt/ic/share/guestos.xml.template) -m=, --media= specify the config media image file (Default: /run/ic-node/config.img) @@ -49,10 +59,28 @@ function validate_arguments() { } # Set arguments if undefined +CONFIG="${CONFIG:=/boot/config/config.ini}" +DEPLOYMENT="${DEPLOYMENT:=/boot/config/deployment.json}" INPUT="${INPUT:=/opt/ic/share/guestos.xml.template}" MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" +# TODO(NODE-1518): remove passing old config +function read_old_config_variables() { + # Read limited set of keys. Be extra-careful quoting values as it could + # otherwise lead to executing arbitrary shell code! + while IFS="=" read -r key value; do + case "$key" in + "ipv6_prefix") ipv6_prefix="${value}" ;; + "ipv6_gateway") ipv6_gateway="${value}" ;; + "ipv4_address") ipv4_address="${value}" ;; + "ipv4_prefix_length") ipv4_prefix_length="${value}" ;; + "ipv4_gateway") ipv4_gateway="${value}" ;; + "domain") domain="${value}" ;; + "node_reward_type") node_reward_type="${value}" ;; + esac + done <"${CONFIG}" + function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') @@ -76,6 +104,19 @@ function assemble_config_media() { cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi + cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") + cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") + cmd+=(--ipv6_gateway "${ipv6_gateway}") + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then + cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") + cmd+=(--ipv4_gateway "${ipv4_gateway}") + cmd+=(--domain "${domain}") + fi + if [[ -n "$node_reward_type" ]]; then + cmd+=(--node_reward_type "${node_reward_type}") + cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") + # Run the above command "${cmd[@]}" write_log "Assembling config media for GuestOS: ${MEDIA}" @@ -115,6 +156,7 @@ function generate_guestos_config() { function main() { validate_arguments + read_old_config_variables read_config_variables assemble_config_media generate_guestos_config diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index e7a1dec346f..8ae7c16ec9a 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -50,6 +50,7 @@ component_files = { Label("misc/config/config.sh"): "/opt/ic/bin/config.sh", Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", Label("misc/metrics.sh"): "/opt/ic/bin/metrics.sh", + Label("misc/fetch-property.sh"): "/opt/ic/bin/fetch-property.sh", Label("misc/vsock/vsock-agent.service"): "/etc/systemd/system/vsock-agent.service", Label("misc/vsock/10-vhost-vsock.rules"): "/etc/udev/rules.d/10-vhost-vsock.rules", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", diff --git a/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh b/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh index 7be1d64f502..20d6189aa08 100755 --- a/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh +++ b/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.sh @@ -100,7 +100,8 @@ function process_bootstrap() { fi done - for FILE in config.json; do + # TODO(NODE-1518): remove parsing for old config + for FILE in config.json filebeat.conf network.conf reward.conf nns.conf backup.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf jaeger_addr.conf socks_proxy.conf; do if [ -e "${TMPDIR}/${FILE}" ]; then echo "Setting up ${FILE}" cp "${TMPDIR}/${FILE}" "${CONFIG_ROOT}/${FILE}" diff --git a/ic-os/components/misc/fetch-property.sh b/ic-os/components/misc/fetch-property.sh new file mode 100644 index 00000000000..14cd519545f --- /dev/null +++ b/ic-os/components/misc/fetch-property.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# Fetch configuration property + +source /opt/ic/bin/logging.sh + +SCRIPT="$(basename $0)[$$]" + +# Get keyword arguments +for argument in "${@}"; do + case ${argument} in + -c=* | --config=*) + CONFIG="${argument#*=}" + shift + ;; + -h | --help) + echo 'Usage: +Fetch Configuration Property +Arguments: + -c=, --config= mandatory: specify the configuration file to read from + -h, --help show this help message and exit + -k=, --key= mandatory: specify the property key + -m=, --metric= optional: specify the metric name (required if metrics.sh exists) +' + exit 1 + ;; + -k=* | --key=*) + KEY="${argument#*=}" + shift + ;; + -m=* | --metric=*) + METRIC="${argument#*=}" + shift + ;; + *) + echo "Error: Argument is not supported." + exit 1 + ;; + esac +done + +function validate_arguments() { + if [ -z "${CONFIG}" ] || [ -z "${KEY}" ]; then + $0 --help + fi + + if [ -f "/opt/ic/bin/metrics.sh" ] && [ -z "${METRIC:-}" ]; then + echo "Error: METRIC is required when metrics.sh exists." + exit 1 + fi +} + +try_write_metric() { + local name=$1 + local value=$2 + local help=$3 + local type=$4 + + # metrics.sh is required for writing metrics + # metrics.sh only exists on HostOS and GuestOS, not SetupOS + if [ -f "/opt/ic/bin/metrics.sh" ]; then + source "/opt/ic/bin/metrics.sh" + write_metric "${name}" "${value}" "${help}" "${type}" + fi +} + +function fetch_property() { + PROPERTY=$(jq -r "$(echo ${KEY})" ${CONFIG}) + + if [ -z "${PROPERTY}" ] || [ "${PROPERTY}" == "null" ]; then + write_log "ERROR: Unable to fetch property: ${KEY}" + try_write_metric "$(echo ${METRIC:-})" \ + "1" \ + "Property: $(echo ${KEY})" \ + "gauge" + exit 1 + else + write_log "Using property: ${PROPERTY}" + try_write_metric "$(echo ${METRIC:-})" \ + "0" \ + "Property: $(echo ${KEY})" \ + "gauge" + echo "${PROPERTY}" + fi +} + +function main() { + # Establish run order + validate_arguments + fetch_property +} + +main From d1b355248fda9b0796c2c9132f2fcf3acc8b0e46 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 19 Nov 2024 22:28:55 +0000 Subject: [PATCH 175/241] fix build-bootstrap formatting --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 957ce14b8ab..d8395175cb3 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -256,7 +256,7 @@ function build_ic_bootstrap_tar() { fi # TODO(NODE-1518): remove parsing for old config - [[ "$HOSTNAME" == "" ]] || [[ "$HOSTNAME" =~ [a-zA-Z]*([a-zA-Z0-9])*(-+([a-zA-Z0-9])) ]] || { + [[ "$HOSTNAME" == "" ]] || [[ "$HOSTNAME" =~ [a-zA-Z]*([a-zA-Z0-9])*(-+([a-zA-Z0-9])) ]] || { echo "Invalid hostname: '$HOSTNAME'" >&2 exit 1 } From 004c7958ea827341969b85cdbd3f67b85ec11849 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 19 Nov 2024 22:30:58 +0000 Subject: [PATCH 176/241] Fix build-bootstrap syntax --- .../components/hostos-scripts/build-bootstrap-config-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index d8395175cb3..9a5f2c662bd 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -261,7 +261,7 @@ function build_ic_bootstrap_tar() { exit 1 } - cat >"${BOOTSTRAP_TMPDIR}/network.conf" <"${BOOTSTRAP_TMPDIR}/network.conf" <>"${BOOTSTRAP_TMPDIR}/filebeat.conf" - if [ "${NNS_URLS}" != "" ]; then echo "nns_url=${NNS_URLS}" >"${BOOTSTRAP_TMPDIR}/nns.conf" fi @@ -299,6 +298,7 @@ EOF if [ "${SOCKS_PROXY}" != "" ]; then echo "socks_proxy=${SOCKS_PROXY}" >"${BOOTSTRAP_TMPDIR}/socks_proxy.conf" + fi tar cf "${OUT_FILE}" \ --sort=name \ From 4f3e848d55e9b3562db34aaad46803e95a1f30ad Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 19 Nov 2024 22:32:59 +0000 Subject: [PATCH 177/241] Fix build-bootsrap syntax --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 9a5f2c662bd..9d356367456 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -295,7 +295,7 @@ EOF fi if [ "${JAEGER_ADDR}" != "" ]; then echo "jaeger_addr=http://${JAEGER_ADDR}" >"${BOOTSTRAP_TMPDIR}/jaeger_addr.conf" - + fi if [ "${SOCKS_PROXY}" != "" ]; then echo "socks_proxy=${SOCKS_PROXY}" >"${BOOTSTRAP_TMPDIR}/socks_proxy.conf" fi From 1a0a69b3e2a79b0a7c5bd6568c5f0287073bb026 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 19 Nov 2024 22:33:55 +0000 Subject: [PATCH 178/241] Fix build-bootsrap syntax --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 9d356367456..a2133e352b1 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -277,6 +277,7 @@ EOF fi if [ "${ELASTICSEARCH_TAGS}" != "" ]; then echo "elasticsearch_tags=$ELASTICSEARCH_TAGS" >>"${BOOTSTRAP_TMPDIR}/filebeat.conf" + fi if [ "${NNS_URLS}" != "" ]; then echo "nns_url=${NNS_URLS}" >"${BOOTSTRAP_TMPDIR}/nns.conf" fi From cba4b83f747dfff673d705df2ddb64ab47a01941 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 19 Nov 2024 22:39:04 +0000 Subject: [PATCH 179/241] Fix lingering issues with passing config to guestos --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 1 + .../generate-guestos-config/dev-generate-guestos-config.sh | 1 + .../generate-guestos-config/generate-guestos-config.sh | 1 + ic-os/components/misc/fetch-property.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index a2133e352b1..b42ec8bedf5 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -93,6 +93,7 @@ options may be specified: --malicious_behavior malicious_behavior A JSON-object that describes the malicious behavior activated on the node. This is only used for testing. + The Json-object corresponds to this Rust-structure: ic_types::malicious_behaviour::MaliciousBehaviour diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index fe4b702ac92..67306809625 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -118,6 +118,7 @@ function assemble_config_media() { fi if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") + fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 859c8978c98..a81e6265541 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -114,6 +114,7 @@ function assemble_config_media() { fi if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") + fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") diff --git a/ic-os/components/misc/fetch-property.sh b/ic-os/components/misc/fetch-property.sh index 14cd519545f..46c6d2a2eeb 100644 --- a/ic-os/components/misc/fetch-property.sh +++ b/ic-os/components/misc/fetch-property.sh @@ -20,6 +20,7 @@ for argument in "${@}"; do -h | --help) echo 'Usage: Fetch Configuration Property + Arguments: -c=, --config= mandatory: specify the configuration file to read from -h, --help show this help message and exit From a55644b0b8b8ce18f7abedbdb7ae1aa078f2c8ec Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 20 Nov 2024 15:46:15 +0000 Subject: [PATCH 180/241] Fix generate-guestos-config syntax --- .../generate-guestos-config/dev-generate-guestos-config.sh | 1 + .../generate-guestos-config/generate-guestos-config.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 67306809625..658841fac4b 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -80,6 +80,7 @@ function read_old_config_variables() { "node_reward_type") node_reward_type="${value}" ;; esac done <"${CONFIG}" +} function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index a81e6265541..ab44a63b83a 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -80,6 +80,7 @@ function read_old_config_variables() { "node_reward_type") node_reward_type="${value}" ;; esac done <"${CONFIG}" +} function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') From 498a8c2dd323e04a51c394a29132fe436cc63653 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 20 Nov 2024 15:53:00 +0000 Subject: [PATCH 181/241] Remove unused generate_ic_boundary_tls_cert field --- .../components/hostos-scripts/build-bootstrap-config-image.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index b42ec8bedf5..9fc449f7df4 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -217,9 +217,6 @@ function build_ic_bootstrap_tar() { --socks_proxy) SOCKS_PROXY="$2" ;; - --generate_ic_boundary_tls_cert) - IC_BOUNDARY_TLS_CERT_DOMAIN_NAME="$2" - ;; *) echo "Unrecognized option: $1" usage From 8fc55e0abe900ea73c85937edd0784ac0bbcab53 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 20 Nov 2024 17:04:14 +0000 Subject: [PATCH 182/241] Fix hostname config --- .../generate-guestos-config/dev-generate-guestos-config.sh | 3 ++- .../generate-guestos-config/generate-guestos-config.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 658841fac4b..703c78ad941 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -120,7 +120,8 @@ function assemble_config_media() { if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi - cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) + cmd+=(--hostname "guest-${MAC_ADDRESS}") cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") # Run the above command diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index ab44a63b83a..3b9c34e98f5 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -116,7 +116,8 @@ function assemble_config_media() { if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi - cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) + cmd+=(--hostname "guest-{$MAC_ADDRESS}") cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") # Run the above command From 74ba483360d58224edcebcdd2e17afca47d87a8c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 20 Nov 2024 18:06:08 +0000 Subject: [PATCH 183/241] Fix hostname old config --- .../dev-generate-guestos-config.sh | 3 +-- .../generate-guestos-config/generate-guestos-config.sh | 3 +-- rs/ic_os/os_tools/hostos_tool/src/main.rs | 8 ++++++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 703c78ad941..658841fac4b 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -120,8 +120,7 @@ function assemble_config_media() { if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi - MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) - cmd+=(--hostname "guest-${MAC_ADDRESS}") + cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") # Run the above command diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 3b9c34e98f5..ab44a63b83a 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -116,8 +116,7 @@ function assemble_config_media() { if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi - MAC_ADDRESS=$(/opt/ic/bin/hostos_tool generate-mac-address --node-type GuestOS) - cmd+=(--hostname "guest-{$MAC_ADDRESS}") + cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") # Run the above command diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 9a709cdf9d3..4bd0ef391da 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -28,6 +28,7 @@ pub enum Commands { #[arg(short, long, default_value = "HostOS")] node_type: String, }, + FetchMacAddress {}, } #[derive(Parser)] @@ -122,6 +123,13 @@ pub fn main() -> Result<()> { println!("{}", generated_mac); Ok(()) } + Some(Commands::FetchMacAddress {}) => { + let hostos_config: HostOSConfig = + deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + + println!("{}", hostos_config.icos_settings.mgmt_mac); + Ok(()) + } None => Err(anyhow!( "No subcommand specified. Run with '--help' for subcommands" )), From ad967b926aa5e97f98c679f6746093bb54e6d25b Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 20 Nov 2024 20:18:53 +0000 Subject: [PATCH 184/241] Rename use_nns_public_key and use_node_operator_private_key --- .../config/src/generate_testnet_config.rs | 16 +++++++-------- rs/ic_os/config/src/lib.rs | 12 +++++------ rs/ic_os/config/src/main.rs | 20 +++++++++---------- rs/ic_os/config/src/types.rs | 8 ++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index c619ccd355e..211eae67b25 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -27,9 +27,9 @@ pub struct GenerateTestnetConfigArgs { pub deployment_environment: Option, pub elasticsearch_hosts: Option, pub elasticsearch_tags: Option, - pub nns_public_key_exists: Option, + pub use_nns_public_key: Option, pub nns_urls: Option>, - pub node_operator_private_key_exists: Option, + pub use_node_operator_private_key: Option, pub use_ssh_authorized_keys: Option, // GuestOSSettings arguments @@ -75,9 +75,9 @@ fn create_guestos_config(config: GenerateTestnetConfigArgs) -> Result Result urls @@ -188,7 +188,7 @@ fn create_guestos_config(config: GenerateTestnetConfigArgs) -> Result vec![Url::parse("https://wiki.internetcomputer.org")?], }; - let node_operator_private_key_exists = node_operator_private_key_exists.unwrap_or(false); + let use_node_operator_private_key = use_node_operator_private_key.unwrap_or(false); let use_ssh_authorized_keys = use_ssh_authorized_keys.unwrap_or(true); @@ -197,9 +197,9 @@ fn create_guestos_config(config: GenerateTestnetConfigArgs) -> Result, #[arg(long)] - pub nns_public_key_exists: Option, + pub use_nns_public_key: Option, #[arg(long)] pub nns_urls: Option>, #[arg(long)] - pub node_operator_private_key_exists: Option, + pub use_node_operator_private_key: Option, #[arg(long)] pub use_ssh_authorized_keys: Option, @@ -146,9 +146,9 @@ pub fn main() -> Result<()> { Some(Commands::CreateSetuposConfig { config_ini_path, deployment_json_path, - nns_public_key_exists, + use_nns_public_key, use_ssh_authorized_keys, - node_operator_private_key_exists, + use_node_operator_private_key, setupos_config_json_path, }) => { // get config.ini settings @@ -225,9 +225,9 @@ pub fn main() -> Result<()> { mgmt_mac, deployment_environment: deployment_json_settings.deployment.name, logging, - nns_public_key_exists, + use_nns_public_key, nns_urls: deployment_json_settings.nns.url.clone(), - node_operator_private_key_exists, + use_node_operator_private_key, use_ssh_authorized_keys, icos_dev_settings: ICOSDevSettings::default(), }; @@ -356,9 +356,9 @@ pub fn main() -> Result<()> { deployment_environment: clap_args.deployment_environment, elasticsearch_hosts: clap_args.elasticsearch_hosts, elasticsearch_tags: clap_args.elasticsearch_tags, - nns_public_key_exists: clap_args.nns_public_key_exists, + use_nns_public_key: clap_args.use_nns_public_key, nns_urls: clap_args.nns_urls, - node_operator_private_key_exists: clap_args.node_operator_private_key_exists, + use_node_operator_private_key: clap_args.use_node_operator_private_key, use_ssh_authorized_keys: clap_args.use_ssh_authorized_keys, inject_ic_crypto: clap_args.inject_ic_crypto, inject_ic_state: clap_args.inject_ic_state, diff --git a/rs/ic_os/config/src/types.rs b/rs/ic_os/config/src/types.rs index 165ebdedc3c..beb58d4e176 100644 --- a/rs/ic_os/config/src/types.rs +++ b/rs/ic_os/config/src/types.rs @@ -69,10 +69,10 @@ pub struct ICOSSettings { /// "mainnet" or "testnet" pub deployment_environment: String, pub logging: Logging, - pub nns_public_key_exists: bool, + pub use_nns_public_key: bool, /// The URL (HTTP) of the NNS node(s). pub nns_urls: Vec, - pub node_operator_private_key_exists: bool, + pub use_node_operator_private_key: bool, /// This ssh keys directory contains individual files named `admin`, `backup`, `readonly`. /// The contents of these files serve as `authorized_keys` for their respective role account. /// This means that, for example, `accounts_ssh_authorized_keys/admin` @@ -209,9 +209,9 @@ mod tests { elasticsearch_hosts: String::new(), elasticsearch_tags: None, }, - nns_public_key_exists: false, + use_nns_public_key: false, nns_urls: vec![], - node_operator_private_key_exists: false, + use_node_operator_private_key: false, use_ssh_authorized_keys: false, icos_dev_settings: ICOSDevSettings::default(), }, From 8bab788992d9ef4ffe12c94be5d279035dcabb33 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 20 Nov 2024 20:33:38 +0000 Subject: [PATCH 185/241] Update references to use_node_operator_private_key and use_nns_public_key --- .../dev-generate-guestos-config.sh | 9 +++++---- .../generate-guestos-config.sh | 8 ++++---- .../setupos-scripts/setup-hostos-config.sh | 14 +++++++------- .../dev_test_tools/launch-single-vm/src/main.rs | 4 ++-- rs/tests/driver/src/driver/bootstrap.rs | 4 ++-- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 658841fac4b..16e130c4b5a 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -85,9 +85,10 @@ function read_old_config_variables() { function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') - nns_public_key_exists=$(get_config_value '.icos_settings.nns_public_key_exists') + use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') - node_operator_private_key_exists=$(get_config_value '.icos_settings.node_operator_private_key_exists') + use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') + use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') @@ -99,10 +100,10 @@ function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--guestos_config "/boot/config/config-guestos.json") - if [[ "${nns_public_key_exists,,}" == "true" ]]; then + if [[ "${use_nns_public_key,,}" == "true" ]]; then cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") fi - if [ -f "$node_operator_private_key_exists" ]; then + if [ -f "$use_node_operator_private_key" ]; then cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi if [[ "${ssh_authorized_keys,,}" == "true" ]]; then diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index ab44a63b83a..44303e46d26 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -85,9 +85,9 @@ function read_old_config_variables() { function read_config_variables() { ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') - nns_public_key_exists=$(get_config_value '.icos_settings.nns_public_key_exists') + use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') - node_operator_private_key_exists=$(get_config_value '.icos_settings.node_operator_private_key_exists') + use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') } @@ -98,10 +98,10 @@ function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--guestos_config "/boot/config/config-guestos.json") - if [[ "${nns_public_key_exists,,}" == "true" ]]; then + if [[ "${use_nns_public_key,,}" == "true" ]]; then cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") fi - if [ -f "$node_operator_private_key_exists" ]; then + if [ -f "$use_node_operator_private_key" ]; then cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi diff --git a/ic-os/components/setupos-scripts/setup-hostos-config.sh b/ic-os/components/setupos-scripts/setup-hostos-config.sh index c2aa501062b..9b817114f26 100755 --- a/ic-os/components/setupos-scripts/setup-hostos-config.sh +++ b/ic-os/components/setupos-scripts/setup-hostos-config.sh @@ -49,29 +49,29 @@ function copy_config_files() { fi echo "* Copying node operator private key..." - node_operator_private_key_exists=$(get_config_value '.icos_settings.node_operator_private_key_exists') - if [[ "${node_operator_private_key_exists,,}" == "true" ]]; then + use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') + if [[ "${use_node_operator_private_key,,}" == "true" ]]; then if [ -f "${CONFIG_DIR}/node_operator_private_key.pem" ]; then cp "${CONFIG_DIR}/node_operator_private_key.pem" /media/ log_and_halt_installation_on_error "${?}" "Unable to copy node operator private key to hostOS config partition." else - log_and_halt_installation_on_error "1" "node_operator_private_key_exists set to true but not found" + log_and_halt_installation_on_error "1" "use_node_operator_private_key set to true but not found" fi else echo >&2 "Warning: node_operator_private_key.pem does not exist, requiring HSM." fi echo "* Copying NNS public key to hostOS config partition..." - nns_public_key_exists=$(get_config_value '.icos_settings.nns_public_key_exists') - if [[ "${nns_public_key_exists,,}" == "true" ]]; then + use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') + if [[ "${use_nns_public_key,,}" == "true" ]]; then if [ -f "/data/nns_public_key.pem" ]; then cp /data/nns_public_key.pem /media/ log_and_halt_installation_on_error "${?}" "Unable to copy NNS public key to hostOS config partition." else - log_and_halt_installation_on_error "1" "nns_public_key_exists set to true but not found." + log_and_halt_installation_on_error "1" "use_nns_public_key set to true but not found." fi else - log_and_halt_installation_on_error "1" "nns_public_key_exists must be set to true." + log_and_halt_installation_on_error "1" "use_nns_public_key must be set to true." fi echo "* Converting 'config.json' to hostOS config file 'config-hostos.json'..." diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index a238f14a2bc..c9e76d3a8bf 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -222,9 +222,9 @@ fn main() { deployment_environment: Some("testnet".to_string()), elasticsearch_hosts: None, elasticsearch_tags: None, - nns_public_key_exists: Some(true), + use_nns_public_key: Some(true), nns_urls: Some(vec![format!("http://[{}]", ipv6_addr)]), - node_operator_private_key_exists: Some(true), + use_node_operator_private_key: Some(true), use_ssh_authorized_keys: Some(true), inject_ic_crypto: Some(false), inject_ic_state: Some(false), diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index c6a95f83a1d..c1aeecb46dc 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -440,9 +440,9 @@ fn create_config_disk_image( deployment_environment: Some("testnet".to_string()), elasticsearch_hosts: None, elasticsearch_tags: Some(format!("system_test {}", group_name)), - nns_public_key_exists: Some(true), + use_nns_public_key: Some(true), nns_urls: None, - node_operator_private_key_exists: Some(true), + use_node_operator_private_key: Some(true), use_ssh_authorized_keys: Some(true), inject_ic_crypto: Some(false), inject_ic_state: Some(false), From 32fa307c8ed8a08d4a2849e343e445c89d23fff1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 22 Nov 2024 21:51:33 +0000 Subject: [PATCH 186/241] Add config_types crate to os_tools --- rs/ic_os/os_tools/hostos_tool/BUILD.bazel | 1 + rs/ic_os/os_tools/hostos_tool/Cargo.toml | 1 + rs/ic_os/os_tools/setupos_tool/BUILD.bazel | 1 + rs/ic_os/os_tools/setupos_tool/Cargo.toml | 1 + 4 files changed, 4 insertions(+) diff --git a/rs/ic_os/os_tools/hostos_tool/BUILD.bazel b/rs/ic_os/os_tools/hostos_tool/BUILD.bazel index 5ae14a2851d..a53d5fa57f8 100644 --- a/rs/ic_os/os_tools/hostos_tool/BUILD.bazel +++ b/rs/ic_os/os_tools/hostos_tool/BUILD.bazel @@ -5,6 +5,7 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", + "//rs/ic_os/config_types", "//rs/ic_os/deterministic_ips", "//rs/ic_os/network", "//rs/ic_os/utils", diff --git a/rs/ic_os/os_tools/hostos_tool/Cargo.toml b/rs/ic_os/os_tools/hostos_tool/Cargo.toml index f7036cb773e..42e7dd27e6d 100644 --- a/rs/ic_os/os_tools/hostos_tool/Cargo.toml +++ b/rs/ic_os/os_tools/hostos_tool/Cargo.toml @@ -11,6 +11,7 @@ path = "src/main.rs" anyhow = { workspace = true } clap = { workspace = true } config = { path = "../../config" } +config_types = { path = "../../config_types" } network = { path = "../../network" } deterministic_ips = { path = "../../deterministic_ips" } utils = { path = "../../utils" } diff --git a/rs/ic_os/os_tools/setupos_tool/BUILD.bazel b/rs/ic_os/os_tools/setupos_tool/BUILD.bazel index b5b76dacfca..d992c4f1bd1 100644 --- a/rs/ic_os/os_tools/setupos_tool/BUILD.bazel +++ b/rs/ic_os/os_tools/setupos_tool/BUILD.bazel @@ -5,6 +5,7 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", + "//rs/ic_os/config_types", "//rs/ic_os/deterministic_ips", "//rs/ic_os/network", "//rs/ic_os/utils", diff --git a/rs/ic_os/os_tools/setupos_tool/Cargo.toml b/rs/ic_os/os_tools/setupos_tool/Cargo.toml index 5f63132abde..31faa749eef 100644 --- a/rs/ic_os/os_tools/setupos_tool/Cargo.toml +++ b/rs/ic_os/os_tools/setupos_tool/Cargo.toml @@ -11,6 +11,7 @@ path = "src/main.rs" anyhow = { workspace = true } clap = { workspace = true } config = { path = "../../config" } +config_types = { path = "../../config_types" } network = { path = "../../network" } deterministic_ips = { path = "../../deterministic_ips" } utils = { path = "../../utils" } From ba38598b7ef9d051f18bfc453c607307b8c8c36e Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Fri, 22 Nov 2024 21:53:05 +0000 Subject: [PATCH 187/241] Automatically updated Cargo*.lock --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f3fa288f5fe..05ca99c5b88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4962,6 +4962,7 @@ dependencies = [ "anyhow", "clap 4.5.20", "config", + "config_types", "deterministic_ips", "network", "utils", @@ -19534,6 +19535,7 @@ dependencies = [ "anyhow", "clap 4.5.20", "config", + "config_types", "deterministic_ips", "network", "utils", From 8e487c10f933433dae44b3cbfb8aedce58e48176 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 19:30:51 +0000 Subject: [PATCH 188/241] Fix hostos and setupos tools --- Cargo.lock | 1 + rs/ic_os/os_tools/hostos_tool/src/main.rs | 20 +++++++++++++++++--- rs/ic_os/os_tools/setupos_tool/src/main.rs | 4 ++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e865c77040d..2639ec48ae8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12703,6 +12703,7 @@ dependencies = [ "canister-test", "chrono", "clap 4.5.20", + "config_types", "crossbeam-channel", "cycles-minting-canister", "deterministic_ips", diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 92bda1eac66..67ff8434b07 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -59,7 +59,7 @@ pub fn main() -> Result<()> { &hostos_config.icos_settings.mgmt_mac, hostos_config.icos_settings.deployment_environment, IpVariant::V6, - 0x0, /* 0x0 corresponds to HostOS */ + NodeType::HostOS, ); generate_network_config( @@ -83,8 +83,22 @@ pub fn main() -> Result<()> { IpVariant::V6, node_type, ); - let ipv6_address = generated_mac.calculate_slaac(&network_info.ipv6_prefix)?; - println!("{}", to_cidr(ipv6_address, network_info.ipv6_subnet)); + + eprintln!("Using generated mac address {}", generated_mac); + + let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = + &hostos_config.network_settings.ipv6_config + { + ipv6_config + } else { + return Err(anyhow!( + "Ipv6Config is not of type Deterministic. Cannot generate IPv6 address." + )); + }; + + let ipv6_address = generated_mac.calculate_slaac(&ipv6_config.prefix)?; + println!("{}", to_cidr(ipv6_address, ipv6_config.prefix_length)); + Ok(()) } Some(Commands::GenerateMacAddress { node_type }) => { diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index fadda805371..4abef18d69b 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -65,7 +65,7 @@ pub fn main() -> Result<()> { IpVariant::V6, NodeType::SetupOS, ); - eprintln!("Using generated mac (unformatted) {}", generated_mac); + eprintln!("Using generated mac {}", generated_mac); generate_network_config( &setupos_config.network_settings, @@ -88,7 +88,7 @@ pub fn main() -> Result<()> { IpVariant::V6, node_type, ); - eprintln!("Using generated mac (unformatted) {}", generated_mac); + eprintln!("Using generated mac address {}", generated_mac); let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = &setupos_config.network_settings.ipv6_config From 09e2a42e5d555ea5c15a7165dc9d1dd638dbfb57 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 19:36:26 +0000 Subject: [PATCH 189/241] FIx guestos tool after merge --- Cargo.lock | 1 + rs/ic_os/os_tools/guestos_tool/BUILD.bazel | 1 + rs/ic_os/os_tools/guestos_tool/Cargo.toml | 2 +- rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs | 4 ++-- rs/ic_os/os_tools/guestos_tool/src/main.rs | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2639ec48ae8..93dd2f6b468 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4599,6 +4599,7 @@ dependencies = [ "anyhow", "clap 4.5.20", "config", + "config_types", "indoc", "itertools 0.12.1", "network", diff --git a/rs/ic_os/os_tools/guestos_tool/BUILD.bazel b/rs/ic_os/os_tools/guestos_tool/BUILD.bazel index 6ce380e64ff..2c6bda41766 100644 --- a/rs/ic_os/os_tools/guestos_tool/BUILD.bazel +++ b/rs/ic_os/os_tools/guestos_tool/BUILD.bazel @@ -5,6 +5,7 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", + "//rs/ic_os/config_types", "//rs/ic_os/network", "//rs/ic_os/utils", "@crate_index//:anyhow", diff --git a/rs/ic_os/os_tools/guestos_tool/Cargo.toml b/rs/ic_os/os_tools/guestos_tool/Cargo.toml index eae4d874005..95c2b61fa0c 100644 --- a/rs/ic_os/os_tools/guestos_tool/Cargo.toml +++ b/rs/ic_os/os_tools/guestos_tool/Cargo.toml @@ -11,9 +11,9 @@ path = "src/main.rs" anyhow = { workspace = true } clap = { workspace = true } config = { path = "../../config" } +config_types = { path = "../../config_types" } indoc = "1.0.9" itertools = { workspace = true } network = { path = "../../network" } regex = { workspace = true } utils = { path = "../../utils" } - diff --git a/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs b/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs index 7440d80f595..d6864efdba1 100644 --- a/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs +++ b/rs/ic_os/os_tools/guestos_tool/src/generate_network_config.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use anyhow::{bail, Context, Result}; -use config::types::Ipv6Config; +use config_types::Ipv6Config; use network::interfaces::{get_interface_name as get_valid_interface_name, get_interface_paths}; use utils::get_command_stdout; @@ -275,7 +275,7 @@ fn is_k8s_testnet() -> Result { #[cfg(test)] mod tests { use super::*; - use config::types::*; + use config_types::*; #[test] fn test_create_network_info_with_valid_ipv6_and_ipv4() { diff --git a/rs/ic_os/os_tools/guestos_tool/src/main.rs b/rs/ic_os/os_tools/guestos_tool/src/main.rs index 977843f4e0f..e46946b245d 100644 --- a/rs/ic_os/os_tools/guestos_tool/src/main.rs +++ b/rs/ic_os/os_tools/guestos_tool/src/main.rs @@ -13,7 +13,7 @@ mod generate_network_config; use generate_network_config::{generate_networkd_config, validate_and_construct_ipv4_address_info}; use config::deserialize_config; -use config::types::GuestOSConfig; +use config_types::GuestOSConfig; use network::systemd::{restart_systemd_networkd, DEFAULT_SYSTEMD_NETWORK_DIR}; #[derive(Subcommand)] From af98dc2758f02e28433a5bf65c0cd1f84d1ad159 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 20:13:32 +0000 Subject: [PATCH 190/241] Fix crate imports in test driver --- rs/tests/driver/BUILD.bazel | 1 + rs/tests/driver/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/rs/tests/driver/BUILD.bazel b/rs/tests/driver/BUILD.bazel index baac372c025..853fc393646 100644 --- a/rs/tests/driver/BUILD.bazel +++ b/rs/tests/driver/BUILD.bazel @@ -47,6 +47,7 @@ rust_library( "//rs/crypto/tree_hash", "//rs/crypto/utils/threshold_sig_der", "//rs/cycles_account_manager", + "//rs/ic_os/config:config_lib", "//rs/ic_os/config_types", "//rs/ic_os/deterministic_ips", "//rs/interfaces", diff --git a/rs/tests/driver/Cargo.toml b/rs/tests/driver/Cargo.toml index a266cf92856..834c39c8d4c 100644 --- a/rs/tests/driver/Cargo.toml +++ b/rs/tests/driver/Cargo.toml @@ -17,6 +17,7 @@ candid = { workspace = true } canister-test = { path = "../../rust_canisters/canister_test" } chrono = { workspace = true } clap = { workspace = true } +config = { path = "../../config" } config_types = { path = "../../ic_os/config_types" } crossbeam-channel = { workspace = true } cycles-minting-canister = { path = "../../nns/cmc" } From 914c73120daed818dc2a9648e58305a9b8f765ad Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 20:16:43 +0000 Subject: [PATCH 191/241] Fix path to config --- Cargo.lock | 1 + rs/tests/driver/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 93dd2f6b468..bd7e9ce54e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12704,6 +12704,7 @@ dependencies = [ "canister-test", "chrono", "clap 4.5.20", + "config", "config_types", "crossbeam-channel", "cycles-minting-canister", diff --git a/rs/tests/driver/Cargo.toml b/rs/tests/driver/Cargo.toml index 834c39c8d4c..a93c470274e 100644 --- a/rs/tests/driver/Cargo.toml +++ b/rs/tests/driver/Cargo.toml @@ -17,7 +17,7 @@ candid = { workspace = true } canister-test = { path = "../../rust_canisters/canister_test" } chrono = { workspace = true } clap = { workspace = true } -config = { path = "../../config" } +config = { path = "../../ic_os/config" } config_types = { path = "../../ic_os/config_types" } crossbeam-channel = { workspace = true } cycles-minting-canister = { path = "../../nns/cmc" } From ef0881ec6a20b85ff84bd21b34acb3d5c08183c7 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 20:25:04 +0000 Subject: [PATCH 192/241] Fix testing --- rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs | 3 ++- rs/tests/driver/src/driver/bootstrap.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs index a1ff3525e34..8f99aabc313 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs +++ b/rs/ic_os/dev_test_tools/launch-single-vm/src/main.rs @@ -26,6 +26,7 @@ use url::Url; use config::generate_testnet_config::{ generate_testnet_config, GenerateTestnetConfigArgs, Ipv6ConfigType, }; +use config_types::DeploymentEnvironment; const FARM_BASE_URL: &str = "https://farm.dfinity.systems"; @@ -211,7 +212,7 @@ fn main() { node_reward_type: None, domain_name: None, mgmt_mac: None, - deployment_environment: Some("testnet".to_string()), + deployment_environment: Some(DeploymentEnvironment::Testnet), elasticsearch_hosts: None, elasticsearch_tags: None, use_nns_public_key: Some(true), diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 3198634b201..245318a2cfc 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -25,6 +25,7 @@ use anyhow::{bail, Result}; use config::generate_testnet_config::{ generate_testnet_config, GenerateTestnetConfigArgs, Ipv6ConfigType, }; +use config_types::DeploymentEnvironment; use ic_base_types::NodeId; use ic_prep_lib::{ internet_computer::{IcConfig, InitializedIc, TopologyConfig}, @@ -437,7 +438,7 @@ fn create_config_disk_image( domain_name: None, node_reward_type: None, mgmt_mac: None, - deployment_environment: Some("testnet".to_string()), + deployment_environment: Some(DeploymentEnvironment::Testnet), elasticsearch_hosts: None, elasticsearch_tags: Some(format!("system_test {}", group_name)), use_nns_public_key: Some(true), From 62e2c2b084f246f3b32a75acc757c35186c67a8d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 20:32:30 +0000 Subject: [PATCH 193/241] Fix launch-single-vm crate imports --- Cargo.lock | 1 + rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel | 1 + rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bd7e9ce54e9..9d57ce49f14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14918,6 +14918,7 @@ version = "0.1.0" dependencies = [ "clap 4.5.20", "config", + "config_types", "ic-prep", "ic-registry-subnet-type", "ic-system-test-driver", diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel b/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel index 06cf195743b..c78a4c37099 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel +++ b/rs/ic_os/dev_test_tools/launch-single-vm/BUILD.bazel @@ -5,6 +5,7 @@ package(default_visibility = ["//rs:ic-os-pkg"]) DEPENDENCIES = [ # Keep sorted. "//rs/ic_os/config:config_lib", + "//rs/ic_os/config_types", "//rs/prep", "//rs/registry/subnet_type", "//rs/tests/driver:ic-system-test-driver", diff --git a/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml b/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml index 8f798f809e6..a62dce80db6 100644 --- a/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml +++ b/rs/ic_os/dev_test_tools/launch-single-vm/Cargo.toml @@ -11,6 +11,7 @@ ic-registry-subnet-type = { path = "../../../registry/subnet_type" } ic-system-test-driver = { path = "../../../tests/driver" } ic-types = { path = "../../../types/types" } config = { path = "../../config" } +config_types = { path = "../../config_types" } clap = { workspace = true } reqwest = { workspace = true } From 4074f6dfc257c72ecd42da3d4d0f3836b4d9a2b6 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 21:12:25 +0000 Subject: [PATCH 194/241] Remove old config parsing in generate-guestos-config --- .../check_file_references.py | 7 +- .../dev-generate-guestos-config.sh | 46 ++------- .../generate-guestos-config.sh | 45 ++------- ic-os/components/hostos.bzl | 1 - ic-os/components/misc/fetch-property.sh | 98 ------------------- 5 files changed, 21 insertions(+), 176 deletions(-) delete mode 100644 ic-os/components/misc/fetch-property.sh diff --git a/ic-os/components/conformance_tests/check_file_references.py b/ic-os/components/conformance_tests/check_file_references.py index 29947eb2e5d..cf5425fe3ca 100755 --- a/ic-os/components/conformance_tests/check_file_references.py +++ b/ic-os/components/conformance_tests/check_file_references.py @@ -11,12 +11,7 @@ import tarfile import tempfile -ALLOWED_UNDECLARED_DEPENDENCIES = { - "ic-os/components/misc/fetch-property.sh": { - # fetch-property.sh checks existence of metrics.sh - "/opt/ic/bin/metrics.sh", - } -} +ALLOWED_UNDECLARED_DEPENDENCIES = {} # Check file patterns /opt/ic/... COMPONENT_FILE_PATTERN = r"/opt/ic/[^\s'\"},)]+" diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 16e130c4b5a..47460897aa9 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -11,21 +11,11 @@ source /opt/ic/bin/config.sh # Get keyword arguments for argument in "${@}"; do case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; - -d=* | --deployment=*) - DEPLOYMENT="${argument#*=}" - shift - ;; -h | --help) echo 'Usage: Generate GuestOS Configuration Arguments: - -c=, --config= specify the config.ini configuration file (Default: /boot/config/config.ini) - -d=, --deployment= specify the deployment.json configuration file (Default: /boot/config/deployment.json) -h, --help show this help message and exit -i=, --input= specify the input template file (Default: /opt/ic/share/guestos.xml.template) -m=, --media= specify the config media image file (Default: /run/ic-node/config.img) @@ -59,36 +49,21 @@ function validate_arguments() { } # Set arguments if undefined -CONFIG="${CONFIG:=/boot/config/config.ini}" -DEPLOYMENT="${DEPLOYMENT:=/boot/config/deployment.json}" INPUT="${INPUT:=/opt/ic/share/guestos.xml.template}" MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" -# TODO(NODE-1518): remove passing old config -function read_old_config_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; - "ipv4_address") ipv4_address="${value}" ;; - "ipv4_prefix_length") ipv4_prefix_length="${value}" ;; - "ipv4_gateway") ipv4_gateway="${value}" ;; - "domain") domain="${value}" ;; - "node_reward_type") node_reward_type="${value}" ;; - esac - done <"${CONFIG}" -} - function read_config_variables() { - ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') + domain_name=$(get_config_value '.network_settings.domain_name') + node_reward_type=$(get_config_value '.icos_settings.node_reward_type') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') - use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') @@ -110,19 +85,19 @@ function assemble_config_media() { cmd+=(--accounts_ssh_authorized_keys "/boot/config/ssh_authorized_keys") fi - cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") + cmd+=(--elasticsearch_hosts "${elasticsearch_hosts}") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") - cmd+=(--domain "${domain}") + cmd+=(--domain "${domain_name}") fi if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") - cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") + cmd+=(--nns_urls "${nns_urls}") # Run the above command "${cmd[@]}" @@ -163,7 +138,6 @@ function generate_guestos_config() { function main() { validate_arguments - read_old_config_variables read_config_variables assemble_config_media generate_guestos_config diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 44303e46d26..74d5f476e9c 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -11,21 +11,11 @@ source /opt/ic/bin/config.sh # Get keyword arguments for argument in "${@}"; do case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; - -d=* | --deployment=*) - DEPLOYMENT="${argument#*=}" - shift - ;; -h | --help) echo 'Usage: Generate GuestOS Configuration Arguments: - -c=, --config= specify the config.ini configuration file (Default: /boot/config/config.ini) - -d=, --deployment= specify the deployment.json configuration file (Default: /boot/config/deployment.json) -h, --help show this help message and exit -i=, --input= specify the input template file (Default: /opt/ic/share/guestos.xml.template) -m=, --media= specify the config media image file (Default: /run/ic-node/config.img) @@ -59,31 +49,17 @@ function validate_arguments() { } # Set arguments if undefined -CONFIG="${CONFIG:=/boot/config/config.ini}" -DEPLOYMENT="${DEPLOYMENT:=/boot/config/deployment.json}" INPUT="${INPUT:=/opt/ic/share/guestos.xml.template}" MEDIA="${MEDIA:=/run/ic-node/config.img}" OUTPUT="${OUTPUT:=/var/lib/libvirt/guestos.xml}" -# TODO(NODE-1518): remove passing old config -function read_old_config_variables() { - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; - "ipv4_address") ipv4_address="${value}" ;; - "ipv4_prefix_length") ipv4_prefix_length="${value}" ;; - "ipv4_gateway") ipv4_gateway="${value}" ;; - "domain") domain="${value}" ;; - "node_reward_type") node_reward_type="${value}" ;; - esac - done <"${CONFIG}" -} - function read_config_variables() { - ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') + ipv6_gateway=$(get_config_value '.network_settings.ipv6_config.Deterministic.gateway') + ipv4_address=$(get_config_value '.network_settings.ipv4_config.address') + ipv4_prefix_length=$(get_config_value '.network_settings.ipv4_config.prefix_length') + ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') + domain_name=$(get_config_value '.network_settings.domain_name') + node_reward_type=$(get_config_value '.icos_settings.node_reward_type') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') @@ -105,19 +81,19 @@ function assemble_config_media() { cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi - cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") + cmd+=(--elasticsearch_hosts "${elasticsearch_hosts}") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") - cmd+=(--domain "${domain}") + cmd+=(--domain "${domain_name}") fi if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") - cmd+=(--nns_urls "$(/opt/ic/bin/fetch-property.sh --key=.nns.url --metric=hostos_nns_url --config=${DEPLOYMENT})") + cmd+=(--nns_urls "${nns_urls}") # Run the above command "${cmd[@]}" @@ -158,7 +134,6 @@ function generate_guestos_config() { function main() { validate_arguments - read_old_config_variables read_config_variables assemble_config_media generate_guestos_config diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 4a87d5c7854..b185764eebc 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -50,7 +50,6 @@ component_files = { Label("misc/config/config.sh"): "/opt/ic/bin/config.sh", Label("misc/logging.sh"): "/opt/ic/bin/logging.sh", Label("misc/metrics.sh"): "/opt/ic/bin/metrics.sh", - Label("misc/fetch-property.sh"): "/opt/ic/bin/fetch-property.sh", Label("misc/vsock/vsock-agent.service"): "/etc/systemd/system/vsock-agent.service", Label("misc/vsock/10-vhost-vsock.rules"): "/etc/udev/rules.d/10-vhost-vsock.rules", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", diff --git a/ic-os/components/misc/fetch-property.sh b/ic-os/components/misc/fetch-property.sh deleted file mode 100644 index 46c6d2a2eeb..00000000000 --- a/ic-os/components/misc/fetch-property.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -# Fetch configuration property - -source /opt/ic/bin/logging.sh - -SCRIPT="$(basename $0)[$$]" - -# Get keyword arguments -for argument in "${@}"; do - case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; - -h | --help) - echo 'Usage: -Fetch Configuration Property - -Arguments: - -c=, --config= mandatory: specify the configuration file to read from - -h, --help show this help message and exit - -k=, --key= mandatory: specify the property key - -m=, --metric= optional: specify the metric name (required if metrics.sh exists) -' - exit 1 - ;; - -k=* | --key=*) - KEY="${argument#*=}" - shift - ;; - -m=* | --metric=*) - METRIC="${argument#*=}" - shift - ;; - *) - echo "Error: Argument is not supported." - exit 1 - ;; - esac -done - -function validate_arguments() { - if [ -z "${CONFIG}" ] || [ -z "${KEY}" ]; then - $0 --help - fi - - if [ -f "/opt/ic/bin/metrics.sh" ] && [ -z "${METRIC:-}" ]; then - echo "Error: METRIC is required when metrics.sh exists." - exit 1 - fi -} - -try_write_metric() { - local name=$1 - local value=$2 - local help=$3 - local type=$4 - - # metrics.sh is required for writing metrics - # metrics.sh only exists on HostOS and GuestOS, not SetupOS - if [ -f "/opt/ic/bin/metrics.sh" ]; then - source "/opt/ic/bin/metrics.sh" - write_metric "${name}" "${value}" "${help}" "${type}" - fi -} - -function fetch_property() { - PROPERTY=$(jq -r "$(echo ${KEY})" ${CONFIG}) - - if [ -z "${PROPERTY}" ] || [ "${PROPERTY}" == "null" ]; then - write_log "ERROR: Unable to fetch property: ${KEY}" - try_write_metric "$(echo ${METRIC:-})" \ - "1" \ - "Property: $(echo ${KEY})" \ - "gauge" - exit 1 - else - write_log "Using property: ${PROPERTY}" - try_write_metric "$(echo ${METRIC:-})" \ - "0" \ - "Property: $(echo ${KEY})" \ - "gauge" - echo "${PROPERTY}" - fi -} - -function main() { - # Establish run order - validate_arguments - fetch_property -} - -main From 6d99f3c46083276bbd501601e6d2dabd2fb8aa7d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 21:20:12 +0000 Subject: [PATCH 195/241] Fix generate-guestos-config --- .../dev-generate-guestos-config.sh | 12 ++++++++---- .../generate-guestos-config.sh | 8 ++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 47460897aa9..23485b01a45 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -61,12 +61,15 @@ function read_config_variables() { domain_name=$(get_config_value '.network_settings.domain_name') node_reward_type=$(get_config_value '.icos_settings.node_reward_type') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') - use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') + mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') + + use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') + use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') + vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') - use_ssh_authorized_keys=$(get_config_value '.icos_settings.use_ssh_authorized_keys') } function assemble_config_media() { @@ -81,10 +84,11 @@ function assemble_config_media() { if [ -f "$use_node_operator_private_key" ]; then cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi - if [[ "${ssh_authorized_keys,,}" == "true" ]]; then + if [[ "${use_ssh_authorized_keys,,}" == "true" ]]; then cmd+=(--accounts_ssh_authorized_keys "/boot/config/ssh_authorized_keys") fi + # TODO(NODE-1518): remove passing old config cmd+=(--elasticsearch_hosts "${elasticsearch_hosts}") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") @@ -96,7 +100,7 @@ function assemble_config_media() { if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi - cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + cmd+=(--hostname "guest-${mgmt_mac//:/}") cmd+=(--nns_urls "${nns_urls}") # Run the above command diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 74d5f476e9c..791acc80119 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -61,9 +61,12 @@ function read_config_variables() { domain_name=$(get_config_value '.network_settings.domain_name') node_reward_type=$(get_config_value '.icos_settings.node_reward_type') elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') - use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') + mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') + + use_nns_public_key=$(get_config_value '.icos_settings.use_nns_public_key') use_node_operator_private_key=$(get_config_value '.icos_settings.use_node_operator_private_key') + vm_memory=$(get_config_value '.hostos_settings.vm_memory') vm_cpu=$(get_config_value '.hostos_settings.vm_cpu') } @@ -81,6 +84,7 @@ function assemble_config_media() { cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi + # TODO(NODE-1518): remove passing old config cmd+=(--elasticsearch_hosts "${elasticsearch_hosts}") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") @@ -92,7 +96,7 @@ function assemble_config_media() { if [[ -n "$node_reward_type" ]]; then cmd+=(--node_reward_type "${node_reward_type}") fi - cmd+=(--hostname "guest-$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g')") + cmd+=(--hostname "guest-${mgmt_mac//:/}") cmd+=(--nns_urls "${nns_urls}") # Run the above command From a8579b3ecc7b8d4822e10eb7597ae4298e9870cd Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 4 Dec 2024 21:28:42 +0000 Subject: [PATCH 196/241] Remove FetchMacAddress command --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 67ff8434b07..4fb82b76333 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -27,7 +27,6 @@ pub enum Commands { #[arg(short, long, default_value_t = NodeType::HostOS)] node_type: NodeType, }, - FetchMacAddress {}, } #[derive(Parser)] @@ -119,13 +118,6 @@ pub fn main() -> Result<()> { println!("{}", generated_mac); Ok(()) } - Some(Commands::FetchMacAddress {}) => { - let hostos_config: HostOSConfig = - deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; - - println!("{}", hostos_config.icos_settings.mgmt_mac); - Ok(()) - } None => Err(anyhow!( "No subcommand specified. Run with '--help' for subcommands" )), From 0390f07de121f7017a2c0686c0c0669298ec432a Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 5 Dec 2024 19:56:17 +0000 Subject: [PATCH 197/241] Update configuration documentation --- ic-os/docs/Configuration.adoc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ic-os/docs/Configuration.adoc b/ic-os/docs/Configuration.adoc index 6cec9171171..5b06903fe92 100644 --- a/ic-os/docs/Configuration.adoc +++ b/ic-os/docs/Configuration.adoc @@ -5,12 +5,10 @@ Each IC-OS has a 100 MB config partition. All IC-OS config partitions are initia In production, configuration is propagated from a partition on the USB installer through each of SetupOS, HostOS and GuestOS. This process is controlled by the (link:../../rs/ic_os/config/README.md[ic-os config tool]) and an assortment of bash scripts. -All access to the config partition should be done through the ic-os config tool and config object. - == User-facing configuration files SetupOS constructs its config struct from the following user-facing configuration files: - config.ini # Data center-specific network settings + config.ini # Node Provider network settings and node settings ssh_authorized_keys # SSH private keys node_operator_private_key.pem # Node Operator private key created in the Node Provider onboarding deployment.json # Deployment-specific configurations @@ -91,7 +89,7 @@ Consider that values may be controlled by an attacker on boot. Bootstrapping a n === Testing For testing, to add new configuration bits, you can modify the config tool located at -link:../../rs/ic_os/config/README.md[rs/ic_os/config]. Or, you may find it easier to update *bootstrap-ic-node.sh* directly, +link:../../rs/ic_os/config/README.md[rs/ic_os/config]. Or, you may find it easier to update *build-bootstrap-config-image.sh* and *bootstrap-ic-node.sh* directly, particularly if you wish to add a new configuration file (as opposed to just a new configuration _field_). * *ic_os config tool* can be run stand-alone to verify that it produces the intended configuration object. From b0e1c8e6c77e95f079ce7e4e9489ea6b92b5b8fe Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 5 Dec 2024 20:51:15 +0000 Subject: [PATCH 198/241] Add back node_reward_type to static testnet --- testnet/tools/build-guestos-configs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/testnet/tools/build-guestos-configs.sh b/testnet/tools/build-guestos-configs.sh index 9c68a291114..a430bfae790 100755 --- a/testnet/tools/build-guestos-configs.sh +++ b/testnet/tools/build-guestos-configs.sh @@ -320,6 +320,7 @@ function build_bootstrap_images() { ${use_crypto:+"--ic_crypto"} ${use_crypto:+"${IC_PREP_DIR}/node-${node_idx}/crypto/"} \ "--nns_urls" "${NNS_URLS}" \ "--nns_public_key" "${IC_PREP_DIR}/nns_public_key.pem" \ + "--node_reward_type type3.1" \ "--hostname" "${hostname}" \ "--accounts_ssh_authorized_keys" "${SSH}" \ ${ELASTICSEARCH_HOSTS:+"--elasticsearch_hosts"} ${ELASTICSEARCH_HOSTS:+"${ELASTICSEARCH_HOSTS}"} \ From d7249e0cf1981b0727d2657182046edaefa8c0b5 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 5 Dec 2024 21:04:09 +0000 Subject: [PATCH 199/241] Add back old config constants --- rs/ic_os/config/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rs/ic_os/config/src/lib.rs b/rs/ic_os/config/src/lib.rs index 312086db394..79461fc000c 100644 --- a/rs/ic_os/config/src/lib.rs +++ b/rs/ic_os/config/src/lib.rs @@ -15,6 +15,9 @@ pub static DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH: &str = "/data/deployment.json"; pub static DEFAULT_SETUPOS_HOSTOS_CONFIG_OBJECT_PATH: &str = "/var/ic/config/config-hostos.json"; +// TODO(NODE-1518): remove unused constants +pub static DEFAULT_HOSTOS_CONFIG_INI_FILE_PATH: &str = "/boot/config/config.ini"; +pub static DEFAULT_HOSTOS_DEPLOYMENT_JSON_PATH: &str = "/boot/config/deployment.json"; pub static DEFAULT_HOSTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config.json"; pub static DEFAULT_HOSTOS_GUESTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config-guestos.json"; pub static DEFAULT_GUESTOS_CONFIG_OBJECT_PATH: &str = "/boot/config/config.json"; From 7f0aac5dfce452ac6c31c71845af1e7cca01ecad Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 5 Dec 2024 21:49:51 +0000 Subject: [PATCH 200/241] Remove duplicate release:config file install --- ic-os/hostos/defs.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/ic-os/hostos/defs.bzl b/ic-os/hostos/defs.bzl index 4d98e2973de..114e6cb361a 100644 --- a/ic-os/hostos/defs.bzl +++ b/ic-os/hostos/defs.bzl @@ -31,7 +31,6 @@ def image_deps(mode, _malicious = False): # additional files to install "//rs/ic_os/release:vsock_host": "/opt/ic/bin/vsock_host:0755", "//rs/ic_os/release:hostos_tool": "/opt/ic/bin/hostos_tool:0755", - "//rs/ic_os/release:config": "/opt/ic/bin/config:0755", "//rs/ic_os/release:metrics-proxy": "/opt/ic/bin/metrics-proxy:0755", "//rs/ic_os/release:config": "/opt/ic/bin/config:0755", From 1c47e79e4670409f6dca46dc77ae144209d79f52 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 6 Dec 2024 22:08:11 +0000 Subject: [PATCH 201/241] Remove mercury targets from unit tests --- rs/ic_os/config/src/deployment_json.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/rs/ic_os/config/src/deployment_json.rs b/rs/ic_os/config/src/deployment_json.rs index f3b9dbbf942..a5590090ca7 100644 --- a/rs/ic_os/config/src/deployment_json.rs +++ b/rs/ic_os/config/src/deployment_json.rs @@ -89,7 +89,7 @@ mod test { "mgmt_mac": null }, "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" + "hosts": "elasticsearch.ch1-obsdev1.dfinity.network:443" }, "nns": { "url": "https://icp-api.io,https://icp0.io,https://ic0.app" @@ -108,7 +108,7 @@ mod test { "mgmt_mac": null }, "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" + "hosts": "elasticsearch.ch1-obsdev1.dfinity.network:443" }, "nns": { "url": "https://icp-api.io,https://icp0.io,https://ic0.app" @@ -120,13 +120,7 @@ mod test { }"#; static DEPLOYMENT_STRUCT: Lazy = Lazy::new(|| { - let hosts = [ - "elasticsearch-node-0.mercury.dfinity.systems:443", - "elasticsearch-node-1.mercury.dfinity.systems:443", - "elasticsearch-node-2.mercury.dfinity.systems:443", - "elasticsearch-node-3.mercury.dfinity.systems:443", - ] - .join(" "); + let hosts = ["elasticsearch.ch1-obsdev1.dfinity.network:443"].join(" "); DeploymentSettings { deployment: Deployment { name: "mainnet".to_string(), From 326bfcff91028f6af9a2bb3a230eb9126a0d907f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 6 Dec 2024 22:54:49 +0000 Subject: [PATCH 202/241] Add node-reward-type default to setupos-inject-config --- rs/tests/driver/src/driver/bootstrap.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 245318a2cfc..487817b7203 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -704,6 +704,8 @@ fn configure_setupos_image( .arg(nns_url.to_string()) .arg("--nns-public-key") .arg(nns_public_key) + .arg("--node-reward-type") + .arg("type3.1") .env(path_key, &new_path); if !admin_keys.is_empty() { From f716d97ea4835d76b4b49e26a06e109197644feb Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 6 Dec 2024 22:52:22 +0000 Subject: [PATCH 203/241] Allow empty hostname in GuestOS --- rs/ic_os/config/src/update_config.rs | 36 +++++++++------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/rs/ic_os/config/src/update_config.rs b/rs/ic_os/config/src/update_config.rs index c5a18fce0b1..3dd5cab900e 100644 --- a/rs/ic_os/config/src/update_config.rs +++ b/rs/ic_os/config/src/update_config.rs @@ -206,19 +206,14 @@ fn read_nns_conf(config_dir: &Path) -> Result> { } fn derive_mgmt_mac_from_hostname(hostname: Option<&str>) -> Result { - if let Some(hostname) = hostname { - if let Some(unformatted_mac) = hostname.strip_prefix("guest-") { - unformatted_mac - .parse() - .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) - } else { - Err(anyhow::anyhow!( - "Hostname does not start with 'guest-': {}", - hostname - )) - } + if let Some(unformatted_mac) = hostname.and_then(|h| h.strip_prefix("guest-")) { + unformatted_mac + .parse() + .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) } else { - Err(anyhow::anyhow!("Hostname is not specified")) + "00:00:00:00:00:00" + .parse() + .map_err(|_| anyhow!("Unable to parse dummy mac address")) } } @@ -392,19 +387,10 @@ mod tests { let mac = derive_mgmt_mac_from_hostname(hostname)?; assert_eq!(mac, expected_mac); - // Test with invalid hostname (wrong prefix) - let invalid_hostname = Some("host-001122334455"); - let result = derive_mgmt_mac_from_hostname(invalid_hostname); - assert!(result.is_err()); - - // Test with invalid hostname (wrong length) - let invalid_hostname_length = Some("guest-00112233"); - let result = derive_mgmt_mac_from_hostname(invalid_hostname_length); - assert!(result.is_err()); - - // Test with None - let result = derive_mgmt_mac_from_hostname(None); - assert!(result.is_err()); + // Test empty hostname + let expected_mac: MacAddr6 = "00:00:00:00:00:00".parse().unwrap(); + let mac = derive_mgmt_mac_from_hostname(None)?; + assert_eq!(mac, expected_mac); Ok(()) } From 79b77e3f868f1134d7c1e23bb3e0e13ff17aa5ef Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 9 Dec 2024 19:12:58 +0000 Subject: [PATCH 204/241] Revert "Allow empty hostname in GuestOS" This reverts commit f716d97ea4835d76b4b49e26a06e109197644feb. --- rs/ic_os/config/src/update_config.rs | 36 +++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/rs/ic_os/config/src/update_config.rs b/rs/ic_os/config/src/update_config.rs index 05fa0e3b4c9..f11b485d5ea 100644 --- a/rs/ic_os/config/src/update_config.rs +++ b/rs/ic_os/config/src/update_config.rs @@ -175,14 +175,19 @@ fn read_nns_conf(config_dir: &Path) -> Result> { } fn derive_mgmt_mac_from_hostname(hostname: Option<&str>) -> Result { - if let Some(unformatted_mac) = hostname.and_then(|h| h.strip_prefix("guest-")) { - unformatted_mac - .parse() - .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) + if let Some(hostname) = hostname { + if let Some(unformatted_mac) = hostname.strip_prefix("guest-") { + unformatted_mac + .parse() + .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) + } else { + Err(anyhow::anyhow!( + "Hostname does not start with 'guest-': {}", + hostname + )) + } } else { - "00:00:00:00:00:00" - .parse() - .map_err(|_| anyhow!("Unable to parse dummy mac address")) + Err(anyhow::anyhow!("Hostname is not specified")) } } @@ -351,10 +356,19 @@ mod tests { let mac = derive_mgmt_mac_from_hostname(hostname)?; assert_eq!(mac, expected_mac); - // Test empty hostname - let expected_mac: MacAddr6 = "00:00:00:00:00:00".parse().unwrap(); - let mac = derive_mgmt_mac_from_hostname(None)?; - assert_eq!(mac, expected_mac); + // Test with invalid hostname (wrong prefix) + let invalid_hostname = Some("host-001122334455"); + let result = derive_mgmt_mac_from_hostname(invalid_hostname); + assert!(result.is_err()); + + // Test with invalid hostname (wrong length) + let invalid_hostname_length = Some("guest-00112233"); + let result = derive_mgmt_mac_from_hostname(invalid_hostname_length); + assert!(result.is_err()); + + // Test with None + let result = derive_mgmt_mac_from_hostname(None); + assert!(result.is_err()); Ok(()) } From a6641e2612c546c131324af72f4a53b892bf3e69 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 9 Dec 2024 19:19:45 +0000 Subject: [PATCH 205/241] Remove unused elasticsearch_hosts variables --- .../generate-guestos-config/dev-generate-guestos-config.sh | 1 - .../generate-guestos-config/generate-guestos-config.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 111a98747e6..5ac88a867ee 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -60,7 +60,6 @@ function read_config_variables() { ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') domain_name=$(get_config_value '.network_settings.domain_name') node_reward_type=$(get_config_value '.icos_settings.node_reward_type') - elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index eab2eafe766..db3520ebcd1 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -60,7 +60,6 @@ function read_config_variables() { ipv4_gateway=$(get_config_value '.network_settings.ipv4_config.gateway') domain_name=$(get_config_value '.network_settings.domain_name') node_reward_type=$(get_config_value '.icos_settings.node_reward_type') - elasticsearch_hosts=$(get_config_value '.icos_settings.logging.elasticsearch_hosts') nns_urls=$(get_config_value '.icos_settings.nns_urls | join(",")') mgmt_mac=$(get_config_value '.icos_settings.mgmt_mac') From 0bbd81073b4a2286eebe55019bbf6dd36c0b4075 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 9 Dec 2024 19:26:05 +0000 Subject: [PATCH 206/241] Revert "Revert "Allow empty hostname in GuestOS"" This reverts commit 79b77e3f868f1134d7c1e23bb3e0e13ff17aa5ef. --- rs/ic_os/config/src/update_config.rs | 36 +++++++++------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/rs/ic_os/config/src/update_config.rs b/rs/ic_os/config/src/update_config.rs index f11b485d5ea..05fa0e3b4c9 100644 --- a/rs/ic_os/config/src/update_config.rs +++ b/rs/ic_os/config/src/update_config.rs @@ -175,19 +175,14 @@ fn read_nns_conf(config_dir: &Path) -> Result> { } fn derive_mgmt_mac_from_hostname(hostname: Option<&str>) -> Result { - if let Some(hostname) = hostname { - if let Some(unformatted_mac) = hostname.strip_prefix("guest-") { - unformatted_mac - .parse() - .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) - } else { - Err(anyhow::anyhow!( - "Hostname does not start with 'guest-': {}", - hostname - )) - } + if let Some(unformatted_mac) = hostname.and_then(|h| h.strip_prefix("guest-")) { + unformatted_mac + .parse() + .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) } else { - Err(anyhow::anyhow!("Hostname is not specified")) + "00:00:00:00:00:00" + .parse() + .map_err(|_| anyhow!("Unable to parse dummy mac address")) } } @@ -356,19 +351,10 @@ mod tests { let mac = derive_mgmt_mac_from_hostname(hostname)?; assert_eq!(mac, expected_mac); - // Test with invalid hostname (wrong prefix) - let invalid_hostname = Some("host-001122334455"); - let result = derive_mgmt_mac_from_hostname(invalid_hostname); - assert!(result.is_err()); - - // Test with invalid hostname (wrong length) - let invalid_hostname_length = Some("guest-00112233"); - let result = derive_mgmt_mac_from_hostname(invalid_hostname_length); - assert!(result.is_err()); - - // Test with None - let result = derive_mgmt_mac_from_hostname(None); - assert!(result.is_err()); + // Test empty hostname + let expected_mac: MacAddr6 = "00:00:00:00:00:00".parse().unwrap(); + let mac = derive_mgmt_mac_from_hostname(None)?; + assert_eq!(mac, expected_mac); Ok(()) } From 89710a2e134b706a83025f3dda3eed4be0cb21ee Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 9 Dec 2024 19:47:37 +0000 Subject: [PATCH 207/241] Keep passing nns_urls to build-bootstrap --- rs/tests/driver/src/driver/bootstrap.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 487817b7203..f376579c12f 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -574,6 +574,17 @@ fn create_config_disk_image( .arg(ssh_authorized_pub_keys_dir); } + // TODO(NODE-1518): remove nns_urls flag, it exists only to pass CI + if let Some(node) = test_env + .topology_snapshot_by_name(ic_name) + .root_subnet() + .nodes() + .next() + { + cmd.arg("--nns_urls") + .arg(format!("http://[{}]:8080", node.get_ip_addr())); + } + let key = "PATH"; let old_path = match std::env::var(key) { Ok(val) => { From be1e88fede731311fb58b5fa85748c3f2633c36f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 9 Dec 2024 20:59:07 +0000 Subject: [PATCH 208/241] Revert "Keep passing nns_urls to build-bootstrap" This reverts commit 89710a2e134b706a83025f3dda3eed4be0cb21ee. --- rs/tests/driver/src/driver/bootstrap.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index f376579c12f..487817b7203 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -574,17 +574,6 @@ fn create_config_disk_image( .arg(ssh_authorized_pub_keys_dir); } - // TODO(NODE-1518): remove nns_urls flag, it exists only to pass CI - if let Some(node) = test_env - .topology_snapshot_by_name(ic_name) - .root_subnet() - .nodes() - .next() - { - cmd.arg("--nns_urls") - .arg(format!("http://[{}]:8080", node.get_ip_addr())); - } - let key = "PATH"; let old_path = match std::env::var(key) { Ok(val) => { From 2c42b36fbcf1e7a9063bd123e22d6e1ed21bf002 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 9 Dec 2024 21:02:28 +0000 Subject: [PATCH 209/241] Add check if new config already exists and is Testnet, do not update --- rs/ic_os/config/src/update_config.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/rs/ic_os/config/src/update_config.rs b/rs/ic_os/config/src/update_config.rs index 05fa0e3b4c9..8fde20a9dc4 100644 --- a/rs/ic_os/config/src/update_config.rs +++ b/rs/ic_os/config/src/update_config.rs @@ -10,7 +10,7 @@ use macaddr::MacAddr6; use crate::config_ini::{get_config_ini_settings, ConfigIniSettings}; use crate::deployment_json::get_deployment_settings; -use crate::serialize_and_write_config; +use crate::{deserialize_config, serialize_and_write_config}; use config_types::*; use network::resolve_mgmt_mac; @@ -26,6 +26,18 @@ pub fn update_guestos_config() -> Result<()> { let network_conf_path = config_dir.join("network.conf"); let config_json_path = config_dir.join("config.json"); + // If a config already exists and is Testnet, do not update. + if config_json_path.exists() { + if let Ok(existing_config) = deserialize_config::(&config_json_path) { + if existing_config.icos_settings.deployment_environment + == DeploymentEnvironment::Testnet + { + println!("A new GuestOSConfig already exists and the environment is Testnet. Skipping update."); + return Ok(()); + } + } + } + let old_config_exists = network_conf_path.exists(); if old_config_exists { @@ -238,6 +250,19 @@ pub fn update_hostos_config( deployment_json_path: &Path, hostos_config_json_path: &PathBuf, ) -> Result<()> { + // If a config already exists and is Testnet, do not update. + if hostos_config_json_path.exists() { + if let Ok(existing_config) = deserialize_config::(&hostos_config_json_path) + { + if existing_config.icos_settings.deployment_environment + == DeploymentEnvironment::Testnet + { + println!("A new HostOSConfig already exists and the environment is Testnet. Skipping update."); + return Ok(()); + } + } + } + let old_config_exists = config_ini_path.exists(); if old_config_exists { From 8a82be7fc40436bcccb7f1842cff56e82dd6491f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 9 Dec 2024 21:03:33 +0000 Subject: [PATCH 210/241] Revert "Revert "Revert "Allow empty hostname in GuestOS""" This reverts commit 0bbd81073b4a2286eebe55019bbf6dd36c0b4075. --- rs/ic_os/config/src/update_config.rs | 36 +++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/rs/ic_os/config/src/update_config.rs b/rs/ic_os/config/src/update_config.rs index 8fde20a9dc4..7d5d9b4fada 100644 --- a/rs/ic_os/config/src/update_config.rs +++ b/rs/ic_os/config/src/update_config.rs @@ -187,14 +187,19 @@ fn read_nns_conf(config_dir: &Path) -> Result> { } fn derive_mgmt_mac_from_hostname(hostname: Option<&str>) -> Result { - if let Some(unformatted_mac) = hostname.and_then(|h| h.strip_prefix("guest-")) { - unformatted_mac - .parse() - .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) + if let Some(hostname) = hostname { + if let Some(unformatted_mac) = hostname.strip_prefix("guest-") { + unformatted_mac + .parse() + .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) + } else { + Err(anyhow::anyhow!( + "Hostname does not start with 'guest-': {}", + hostname + )) + } } else { - "00:00:00:00:00:00" - .parse() - .map_err(|_| anyhow!("Unable to parse dummy mac address")) + Err(anyhow::anyhow!("Hostname is not specified")) } } @@ -376,10 +381,19 @@ mod tests { let mac = derive_mgmt_mac_from_hostname(hostname)?; assert_eq!(mac, expected_mac); - // Test empty hostname - let expected_mac: MacAddr6 = "00:00:00:00:00:00".parse().unwrap(); - let mac = derive_mgmt_mac_from_hostname(None)?; - assert_eq!(mac, expected_mac); + // Test with invalid hostname (wrong prefix) + let invalid_hostname = Some("host-001122334455"); + let result = derive_mgmt_mac_from_hostname(invalid_hostname); + assert!(result.is_err()); + + // Test with invalid hostname (wrong length) + let invalid_hostname_length = Some("guest-00112233"); + let result = derive_mgmt_mac_from_hostname(invalid_hostname_length); + assert!(result.is_err()); + + // Test with None + let result = derive_mgmt_mac_from_hostname(None); + assert!(result.is_err()); Ok(()) } From 285bfad06c0c5e1eedea3e0be1e0f230d2275b1d Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 6 Dec 2024 22:52:22 +0000 Subject: [PATCH 211/241] Allow empty hostname in GuestOS --- rs/ic_os/config/src/update_config.rs | 36 +++++++++------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/rs/ic_os/config/src/update_config.rs b/rs/ic_os/config/src/update_config.rs index 7d5d9b4fada..8fde20a9dc4 100644 --- a/rs/ic_os/config/src/update_config.rs +++ b/rs/ic_os/config/src/update_config.rs @@ -187,19 +187,14 @@ fn read_nns_conf(config_dir: &Path) -> Result> { } fn derive_mgmt_mac_from_hostname(hostname: Option<&str>) -> Result { - if let Some(hostname) = hostname { - if let Some(unformatted_mac) = hostname.strip_prefix("guest-") { - unformatted_mac - .parse() - .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) - } else { - Err(anyhow::anyhow!( - "Hostname does not start with 'guest-': {}", - hostname - )) - } + if let Some(unformatted_mac) = hostname.and_then(|h| h.strip_prefix("guest-")) { + unformatted_mac + .parse() + .map_err(|_| anyhow!("Unable to parse mac address: {}", unformatted_mac)) } else { - Err(anyhow::anyhow!("Hostname is not specified")) + "00:00:00:00:00:00" + .parse() + .map_err(|_| anyhow!("Unable to parse dummy mac address")) } } @@ -381,19 +376,10 @@ mod tests { let mac = derive_mgmt_mac_from_hostname(hostname)?; assert_eq!(mac, expected_mac); - // Test with invalid hostname (wrong prefix) - let invalid_hostname = Some("host-001122334455"); - let result = derive_mgmt_mac_from_hostname(invalid_hostname); - assert!(result.is_err()); - - // Test with invalid hostname (wrong length) - let invalid_hostname_length = Some("guest-00112233"); - let result = derive_mgmt_mac_from_hostname(invalid_hostname_length); - assert!(result.is_err()); - - // Test with None - let result = derive_mgmt_mac_from_hostname(None); - assert!(result.is_err()); + // Test empty hostname + let expected_mac: MacAddr6 = "00:00:00:00:00:00".parse().unwrap(); + let mac = derive_mgmt_mac_from_hostname(None)?; + assert_eq!(mac, expected_mac); Ok(()) } From 4bb974f1d4f6646f25717e4580df073d0cd10038 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 10 Dec 2024 21:33:41 +0000 Subject: [PATCH 212/241] Fix generate-guestos-config.sh --- .../generate-guestos-config/dev-generate-guestos-config.sh | 2 +- .../generate-guestos-config/generate-guestos-config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 5ac88a867ee..694bb13aeba 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -80,7 +80,7 @@ function assemble_config_media() { if [[ "${use_nns_public_key,,}" == "true" ]]; then cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") fi - if [ -f "$use_node_operator_private_key" ]; then + if [[ "${use_node_operator_private_key,,}" == "true" ]]; then cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi if [[ "${use_ssh_authorized_keys,,}" == "true" ]]; then diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index db3520ebcd1..89ea53fa8b4 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -79,7 +79,7 @@ function assemble_config_media() { if [[ "${use_nns_public_key,,}" == "true" ]]; then cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") fi - if [ -f "$use_node_operator_private_key" ]; then + if [[ "${use_node_operator_private_key,,}" == "true" ]]; then cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") From 0e8df984239c7eea788fa5152a98a6578d371392 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 10 Dec 2024 21:34:36 +0000 Subject: [PATCH 213/241] Set testnet deployment environment for nested tests --- .../setupos-inject-configuration/src/main.rs | 7 +++++++ rs/tests/driver/src/driver/bootstrap.rs | 2 ++ 2 files changed, 9 insertions(+) diff --git a/rs/ic_os/dev_test_tools/setupos-inject-configuration/src/main.rs b/rs/ic_os/dev_test_tools/setupos-inject-configuration/src/main.rs index 1dafa3355be..7ae329c676b 100644 --- a/rs/ic_os/dev_test_tools/setupos-inject-configuration/src/main.rs +++ b/rs/ic_os/dev_test_tools/setupos-inject-configuration/src/main.rs @@ -80,6 +80,9 @@ struct DeploymentConfig { #[arg(long)] mgmt_mac: Option, + + #[arg(long)] + deployment_environment: Option, } #[tokio::main] @@ -256,6 +259,10 @@ async fn update_deployment(path: &Path, cfg: &DeploymentConfig) -> Result<(), Er deployment_json.resources.cpu = Some(cpu.to_owned()); } + if let Some(deployment_environment) = &cfg.deployment_environment { + deployment_json.deployment.name = deployment_environment.to_owned(); + } + let mut f = File::create(path).context("failed to open deployment config file")?; let output = serde_json::to_string_pretty(&deployment_json)?; write!(&mut f, "{output}")?; diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 487817b7203..049d7176f7c 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -690,6 +690,8 @@ fn configure_setupos_image( let mut cmd = Command::new(setupos_inject_configs); cmd.arg("--image-path") .arg(&uncompressed_image) + .arg("--deployment-environment") + .arg("Testnet") .arg("--mgmt-mac") .arg(&mac) .arg("--ipv6-prefix") From 2a06def4ac3da9c375f36f98fcadd924bc0e8be9 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 10 Dec 2024 21:49:11 +0000 Subject: [PATCH 214/241] Fix service ordering --- .../early-boot/setup-hostname/hostos/setup-hostname.service | 2 ++ .../components/early-boot/setup-hostname/setup-hostname.service | 2 ++ .../components/ic/generate-ic-config/generate-ic-config.service | 2 ++ .../components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service | 2 ++ .../components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service | 2 ++ .../ic-https-outcalls-adapter/ic-https-outcalls-adapter.service | 2 ++ ic-os/components/misc/log-config/log-config-guestos.service | 2 ++ ic-os/components/misc/log-config/log-config.service | 2 ++ ic-os/components/monitoring/filebeat/filebeat.service | 2 ++ .../networking/nftables/hostos/setup-nftables.service | 2 ++ 10 files changed, 20 insertions(+) diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service index c9dcdab46bc..886fa96375b 100644 --- a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service +++ b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service @@ -5,6 +5,8 @@ Before=systemd-networkd.service After=systemd-tmpfiles-setup.service Wants=dev-ipmi0.device After=dev-ipmi0.device +After=update-hostos-config.service +Wants=update-hostos-config.service [Service] Type=oneshot diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.service b/ic-os/components/early-boot/setup-hostname/setup-hostname.service index 176bc4d806b..6062bd46065 100644 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.service +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.service @@ -3,6 +3,8 @@ Description=Set up hostname Before=systemd-networkd.service DefaultDependencies=no After=bootstrap-ic-node.service +After=update-guestos-config.service +Wants=update-guestos-config.service [Install] WantedBy=multi-user.target diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.service b/ic-os/components/ic/generate-ic-config/generate-ic-config.service index 72c618efa97..cc39de60104 100644 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.service +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.service @@ -4,6 +4,8 @@ Description=Generate IC Configuration # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service +After=update-guestos-config.service +Wants=update-guestos-config.service # We must also wait for storage permission fixup to have finished. After=setup-permissions.service Wants=setup-permissions.service diff --git a/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service b/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service index d90aed38d54..f851f3550d4 100644 --- a/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service +++ b/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service @@ -4,6 +4,8 @@ Description=IC Bitcoin Mainnet Adapter Service Provider # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service +After=update-guestos-config.service +Wants=update-guestos-config.service After=network-online.target Wants=network-online.target Requires=ic-btc-mainnet-adapter.socket diff --git a/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service b/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service index 26f622d4b27..470be473096 100644 --- a/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service +++ b/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service @@ -4,6 +4,8 @@ Description=IC Bitcoin Testnet Adapter Service Provider # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service +After=update-guestos-config.service +Wants=update-guestos-config.service After=network-online.target Wants=network-online.target Requires=ic-btc-testnet-adapter.socket diff --git a/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service b/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service index 244b1eab8cd..d1c7c569121 100644 --- a/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service +++ b/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service @@ -4,6 +4,8 @@ Description=IC Canister HTTP Provider # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service +After=update-guestos-config.service +Wants=update-guestos-config.service After=network-online.target Wants=network-online.target Requires=ic-https-outcalls-adapter.socket diff --git a/ic-os/components/misc/log-config/log-config-guestos.service b/ic-os/components/misc/log-config/log-config-guestos.service index b5ee83f0bd0..01c8de6fe3e 100644 --- a/ic-os/components/misc/log-config/log-config-guestos.service +++ b/ic-os/components/misc/log-config/log-config-guestos.service @@ -2,6 +2,8 @@ Description=Log config partition After=bootstrap-ic-node.service Requires=bootstrap-ic-node.service +After=update-guestos-config.service +Wants=update-guestos-config.service [Service] Type=oneshot diff --git a/ic-os/components/misc/log-config/log-config.service b/ic-os/components/misc/log-config/log-config.service index 24f8ef30fc2..e028238c15b 100644 --- a/ic-os/components/misc/log-config/log-config.service +++ b/ic-os/components/misc/log-config/log-config.service @@ -1,5 +1,7 @@ [Unit] Description=Log config partition +After=update-hostos-config.service +Wants=update-hostos-config.service [Service] Type=oneshot diff --git a/ic-os/components/monitoring/filebeat/filebeat.service b/ic-os/components/monitoring/filebeat/filebeat.service index 8707e255810..65d8738ea93 100644 --- a/ic-os/components/monitoring/filebeat/filebeat.service +++ b/ic-os/components/monitoring/filebeat/filebeat.service @@ -7,6 +7,8 @@ After=network-online.target # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service +After=update-guestos-config.service +Wants=update-guestos-config.service # We must wait for var to be mounted over before interacting with it After=var.mount Wants=var.mount diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.service b/ic-os/components/networking/nftables/hostos/setup-nftables.service index 00237de5698..e2561498c43 100644 --- a/ic-os/components/networking/nftables/hostos/setup-nftables.service +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.service @@ -4,6 +4,8 @@ Description=Generate Firewall Configuration # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service +After=update-hostos-config.service +Wants=update-hostos-config.service # We must also wait for storage permission fixup to have finished. After=setup-permissions.service Wants=setup-permissions.service From 31a50bbb84bc8eb14726a4b270c0538aece97507 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 10 Dec 2024 22:08:48 +0000 Subject: [PATCH 215/241] Update setup-nftables to use new config --- .../nftables/hostos/setup-nftables.service | 2 +- .../nftables/hostos/setup-nftables.sh | 39 +++++-------------- 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.service b/ic-os/components/networking/nftables/hostos/setup-nftables.service index e2561498c43..70130940395 100644 --- a/ic-os/components/networking/nftables/hostos/setup-nftables.service +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.service @@ -14,7 +14,7 @@ Wants=nftables.service [Service] Type=oneshot -ExecStart=/opt/ic/bin/setup-nftables.sh -n /boot/config/config.ini -i /opt/ic/share/nftables.template -o /run/ic-node/nftables-ruleset/nftables.conf +ExecStart=/opt/ic/bin/setup-nftables.sh -i /opt/ic/share/nftables.template -o /run/ic-node/nftables-ruleset/nftables.conf [Install] WantedBy=multi-user.target diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.sh b/ic-os/components/networking/nftables/hostos/setup-nftables.sh index a35a46aa7ed..ea8350208ed 100755 --- a/ic-os/components/networking/nftables/hostos/setup-nftables.sh +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.sh @@ -2,43 +2,28 @@ # Substitute correct configuration parameters into nftables.conf. +source /opt/ic/bin/config.sh + function usage() { cat < Date: Tue, 10 Dec 2024 22:10:31 +0000 Subject: [PATCH 216/241] Fix ipv6_prefix get_config_value --- ic-os/components/networking/nftables/hostos/setup-nftables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.sh b/ic-os/components/networking/nftables/hostos/setup-nftables.sh index ea8350208ed..c8927364fcd 100755 --- a/ic-os/components/networking/nftables/hostos/setup-nftables.sh +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.sh @@ -17,7 +17,7 @@ EOF } function read_config_variables() { - ipv6_prefix=$(get_config_value '.network_settings.prefix.') + ipv6_prefix=$(get_config_value '.network_settings.ipv6_config.Deterministic.prefix') IPV6_PREFIX="${ipv6_prefix:+${ipv6_prefix}::/64}" # Add suffix to prefix if found IPV6_PREFIX="${IPV6_PREFIX:-::1/128}" # Default to loopback for easy templating } From af902d9793c81e316378855f519eae0c999c164c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 11 Dec 2024 17:44:06 +0000 Subject: [PATCH 217/241] Fix HSM deployment bug --- rs/ic_os/config/src/main.rs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/rs/ic_os/config/src/main.rs b/rs/ic_os/config/src/main.rs index 5ef17aac50a..4b9b27240d9 100644 --- a/rs/ic_os/config/src/main.rs +++ b/rs/ic_os/config/src/main.rs @@ -26,15 +26,6 @@ pub enum Commands { #[arg(long, default_value = config::DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH, value_name = "deployment.json")] deployment_json_path: PathBuf, - #[arg(long, default_value_t = true)] - use_nns_public_key: bool, - - #[arg(long, default_value_t = false)] - use_ssh_authorized_keys: bool, - - #[arg(long, default_value_t = true)] - use_node_operator_private_key: bool, - #[arg(long, default_value = config::DEFAULT_SETUPOS_CONFIG_OBJECT_PATH, value_name = "config.json")] setupos_config_json_path: PathBuf, }, @@ -160,9 +151,6 @@ pub fn main() -> Result<()> { Some(Commands::CreateSetuposConfig { config_ini_path, deployment_json_path, - use_nns_public_key, - use_ssh_authorized_keys, - use_node_operator_private_key, setupos_config_json_path, }) => { // get config.ini settings @@ -224,10 +212,11 @@ pub fn main() -> Result<()> { mgmt_mac, deployment_environment: deployment_json_settings.deployment.name.parse()?, logging: Logging::default(), - use_nns_public_key, + use_nns_public_key: Path::new("/data/nns_public_key.pem").exists(), nns_urls: deployment_json_settings.nns.url.clone(), - use_node_operator_private_key, - use_ssh_authorized_keys, + use_node_operator_private_key: Path::new("/config/node_operator_private_key.pem") + .exists(), + use_ssh_authorized_keys: Path::new("/config/ssh_authorized_keys").exists(), icos_dev_settings: ICOSDevSettings::default(), }; From 93ec080e00d258707aa56bcf05d51a7f58aea2e9 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 11 Dec 2024 17:44:26 +0000 Subject: [PATCH 218/241] Update nested test DeploymentEnvironment --- rs/tests/driver/src/driver/nested.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/tests/driver/src/driver/nested.rs b/rs/tests/driver/src/driver/nested.rs index e6bec284501..629d512670d 100644 --- a/rs/tests/driver/src/driver/nested.rs +++ b/rs/tests/driver/src/driver/nested.rs @@ -140,13 +140,13 @@ impl NestedVms for TestEnv { let host_mac = calculate_deterministic_mac( &seed_mac, - DeploymentEnvironment::Mainnet, + DeploymentEnvironment::Testnet, IpVariant::V6, NodeType::HostOS, ); let guest_mac = calculate_deterministic_mac( &seed_mac, - DeploymentEnvironment::Mainnet, + DeploymentEnvironment::Testnet, IpVariant::V6, NodeType::GuestOS, ); From 6fbd20fba5cd725ad7d51d5bcf93b25cf10f0d80 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 11 Dec 2024 21:08:16 +0000 Subject: [PATCH 219/241] Fix service dependencies --- .../early-boot/setup-hostname/hostos/setup-hostname.service | 4 ++-- .../early-boot/setup-hostname/setup-hostname.service | 4 ++-- .../ic/generate-ic-config/generate-ic-config.service | 4 ++-- .../ic/ic-btc-adapter/ic-btc-mainnet-adapter.service | 4 ++-- .../ic/ic-btc-adapter/ic-btc-testnet-adapter.service | 4 ++-- .../ic-https-outcalls-adapter.service | 4 ++-- ic-os/components/misc/log-config/log-config-guestos.service | 4 ++-- ic-os/components/misc/log-config/log-config.service | 4 ++-- ic-os/components/monitoring/filebeat/filebeat.service | 4 ++-- .../guestos/generate-network-config.service | 2 ++ .../hostos/generate-network-config.service | 2 ++ .../networking/nftables/hostos/setup-nftables.service | 4 ++-- 12 files changed, 24 insertions(+), 20 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service index 886fa96375b..9e932f459d9 100644 --- a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service +++ b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.service @@ -5,8 +5,8 @@ Before=systemd-networkd.service After=systemd-tmpfiles-setup.service Wants=dev-ipmi0.device After=dev-ipmi0.device -After=update-hostos-config.service -Wants=update-hostos-config.service +After=update-config.service +Wants=update-config.service [Service] Type=oneshot diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.service b/ic-os/components/early-boot/setup-hostname/setup-hostname.service index 6062bd46065..7655e284632 100644 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.service +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.service @@ -3,8 +3,8 @@ Description=Set up hostname Before=systemd-networkd.service DefaultDependencies=no After=bootstrap-ic-node.service -After=update-guestos-config.service -Wants=update-guestos-config.service +After=update-config.service +Wants=update-config.service [Install] WantedBy=multi-user.target diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.service b/ic-os/components/ic/generate-ic-config/generate-ic-config.service index cc39de60104..d206fde7ebd 100644 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.service +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.service @@ -4,8 +4,8 @@ Description=Generate IC Configuration # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service -After=update-guestos-config.service -Wants=update-guestos-config.service +After=update-config.service +Wants=update-config.service # We must also wait for storage permission fixup to have finished. After=setup-permissions.service Wants=setup-permissions.service diff --git a/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service b/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service index f851f3550d4..24b451aea66 100644 --- a/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service +++ b/ic-os/components/ic/ic-btc-adapter/ic-btc-mainnet-adapter.service @@ -4,8 +4,8 @@ Description=IC Bitcoin Mainnet Adapter Service Provider # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service -After=update-guestos-config.service -Wants=update-guestos-config.service +After=update-config.service +Wants=update-config.service After=network-online.target Wants=network-online.target Requires=ic-btc-mainnet-adapter.socket diff --git a/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service b/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service index 470be473096..2517449f76d 100644 --- a/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service +++ b/ic-os/components/ic/ic-btc-adapter/ic-btc-testnet-adapter.service @@ -4,8 +4,8 @@ Description=IC Bitcoin Testnet Adapter Service Provider # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service -After=update-guestos-config.service -Wants=update-guestos-config.service +After=update-config.service +Wants=update-config.service After=network-online.target Wants=network-online.target Requires=ic-btc-testnet-adapter.socket diff --git a/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service b/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service index d1c7c569121..15404d31620 100644 --- a/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service +++ b/ic-os/components/ic/ic-https-outcalls-adapter/ic-https-outcalls-adapter.service @@ -4,8 +4,8 @@ Description=IC Canister HTTP Provider # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service -After=update-guestos-config.service -Wants=update-guestos-config.service +After=update-config.service +Wants=update-config.service After=network-online.target Wants=network-online.target Requires=ic-https-outcalls-adapter.socket diff --git a/ic-os/components/misc/log-config/log-config-guestos.service b/ic-os/components/misc/log-config/log-config-guestos.service index 01c8de6fe3e..705bdfb511c 100644 --- a/ic-os/components/misc/log-config/log-config-guestos.service +++ b/ic-os/components/misc/log-config/log-config-guestos.service @@ -2,8 +2,8 @@ Description=Log config partition After=bootstrap-ic-node.service Requires=bootstrap-ic-node.service -After=update-guestos-config.service -Wants=update-guestos-config.service +After=update-config.service +Wants=update-config.service [Service] Type=oneshot diff --git a/ic-os/components/misc/log-config/log-config.service b/ic-os/components/misc/log-config/log-config.service index e028238c15b..90bb67bbe41 100644 --- a/ic-os/components/misc/log-config/log-config.service +++ b/ic-os/components/misc/log-config/log-config.service @@ -1,7 +1,7 @@ [Unit] Description=Log config partition -After=update-hostos-config.service -Wants=update-hostos-config.service +After=update-config.service +Wants=update-config.service [Service] Type=oneshot diff --git a/ic-os/components/monitoring/filebeat/filebeat.service b/ic-os/components/monitoring/filebeat/filebeat.service index 65d8738ea93..b8e4da99063 100644 --- a/ic-os/components/monitoring/filebeat/filebeat.service +++ b/ic-os/components/monitoring/filebeat/filebeat.service @@ -7,8 +7,8 @@ After=network-online.target # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service -After=update-guestos-config.service -Wants=update-guestos-config.service +After=update-config.service +Wants=update-config.service # We must wait for var to be mounted over before interacting with it After=var.mount Wants=var.mount diff --git a/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service b/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service index 33f6fc57ef4..df7983797e4 100644 --- a/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service +++ b/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service @@ -2,6 +2,8 @@ Description=Generate network config After=bootstrap-ic-node.service Requires=bootstrap-ic-node.service +After=update-config.service +Wants=update-config.service Before=systemd-networkd.service [Install] diff --git a/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service b/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service index 5db00e88ea4..915bb84d365 100644 --- a/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service +++ b/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service @@ -3,6 +3,8 @@ Description=Generate network config After=systemd-modules-load.service After=systemd-udev-settle.service Wants=systemd-udev-settle.service +After=update-config.service +Wants=update-config.service Before=systemd-networkd.service Before=systemd-networkd-wait-online.service diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.service b/ic-os/components/networking/nftables/hostos/setup-nftables.service index 70130940395..e78ce3cfa07 100644 --- a/ic-os/components/networking/nftables/hostos/setup-nftables.service +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.service @@ -4,8 +4,8 @@ Description=Generate Firewall Configuration # state files and may also be needed to obtain network config. After=bootstrap-ic-node.service Wants=bootstrap-ic-node.service -After=update-hostos-config.service -Wants=update-hostos-config.service +After=update-config.service +Wants=update-config.service # We must also wait for storage permission fixup to have finished. After=setup-permissions.service Wants=setup-permissions.service From c893839fd2924b20ea3bc9244f1aa1a960a68a6e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 11 Dec 2024 21:08:32 +0000 Subject: [PATCH 220/241] Fix documentation --- rs/ic_os/os_tools/guestos_tool/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/ic_os/os_tools/guestos_tool/src/main.rs b/rs/ic_os/os_tools/guestos_tool/src/main.rs index e46946b245d..ebdd7a7e60d 100644 --- a/rs/ic_os/os_tools/guestos_tool/src/main.rs +++ b/rs/ic_os/os_tools/guestos_tool/src/main.rs @@ -25,7 +25,7 @@ pub enum Commands { systemd_network_dir: String, #[arg(long, default_value = config::DEFAULT_GUESTOS_CONFIG_OBJECT_PATH, value_name = "FILE")] - /// network.conf input file + /// config.json input file config_object: PathBuf, }, /// Regenerate systemd network configuration files, optionally incorporating specified IPv4 configuration parameters, and then restart the systemd network. @@ -35,7 +35,7 @@ pub enum Commands { systemd_network_dir: String, #[arg(long, default_value = config::DEFAULT_GUESTOS_CONFIG_OBJECT_PATH, value_name = "FILE")] - /// network.conf input file + /// config.json input file config_object: PathBuf, #[arg(long, value_name = "IPV4_ADDRESS")] From 730665f1155bbf3a36ec03965bda69c13c495097 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 11 Dec 2024 21:08:51 +0000 Subject: [PATCH 221/241] Pass old config to pass downgrade tests --- rs/tests/driver/src/driver/bootstrap.rs | 60 +++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 049d7176f7c..462c0682980 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -477,7 +477,7 @@ fn create_config_disk_image( config.nns_urls = Some(vec![format!("http://[{}]:8080", node.get_ip_addr())]); } - if let Some(malicious_behavior) = malicious_behavior { + if let Some(ref malicious_behavior) = malicious_behavior { info!( test_env.logger(), "Node with id={} has malicious behavior={:?}", node.node_id, malicious_behavior @@ -495,7 +495,7 @@ fn create_config_disk_image( config.query_stats_epoch_length = Some(query_stats_epoch_length); } - if let Some(ipv4_config) = ipv4_config { + if let Some(ref ipv4_config) = ipv4_config { info!( test_env.logger(), "Node with id={} is IPv4-enabled: {:?}", node.node_id, ipv4_config @@ -511,12 +511,12 @@ fn create_config_disk_image( config.generate_ic_boundary_tls_cert = Some(domain_name.to_string()); } - if let Some(domain_name) = domain_name { + if let Some(ref domain_name) = domain_name { info!( test_env.logger(), "Node with id={} has domain_name {}", node.node_id, domain_name, ); - config.domain_name = Some(domain_name); + config.domain_name = Some(domain_name.to_string()); } let elasticsearch_hosts: Vec = get_elasticsearch_hosts()?; @@ -574,6 +574,58 @@ fn create_config_disk_image( .arg(ssh_authorized_pub_keys_dir); } + // TODO(NODE-1518): remove passing old config (only exists to pass *downgrade* CI tests) + if InfraProvider::read_attribute(test_env) == InfraProvider::K8s { + cmd.arg("--ipv6_address") + .arg(format!("{}/64", node.node_config.public_api.ip())) + .arg("--ipv6_gateway") + .arg("fe80::ecee:eeff:feee:eeee"); + } + if let Some(node) = test_env + .topology_snapshot_by_name(ic_name) + .root_subnet() + .nodes() + .next() + { + cmd.arg("--nns_urls") + .arg(format!("http://[{}]:8080", node.get_ip_addr())); + } + if let Some(malicious_behavior) = malicious_behavior { + cmd.arg("--malicious_behavior") + .arg(serde_json::to_string(&malicious_behavior)?); + } + if let Some(query_stats_epoch_length) = query_stats_epoch_length { + cmd.arg("--query_stats_epoch_length") + .arg(format!("{}", query_stats_epoch_length)); + } + if let Some(ipv4_config) = ipv4_config { + cmd.arg("--ipv4_address").arg(format!( + "{}/{:?}", + ipv4_config.ip_addr(), + ipv4_config.prefix_length() + )); + cmd.arg("--ipv4_gateway").arg(ipv4_config.gateway_ip_addr()); + } + if let Some(domain_name) = &node.node_config.domain { + cmd.arg("--generate_ic_boundary_tls_cert").arg(domain_name); + } + if let Some(domain_name) = domain_name { + cmd.arg("--domain").arg(domain_name); + } + if !elasticsearch_hosts.is_empty() { + cmd.arg("--elasticsearch_hosts") + .arg(elasticsearch_hosts.join(" ")); + } + if let Ok(arg) = test_env.read_json_object::(BITCOIND_ADDR_PATH) { + cmd.arg("--bitcoind_addr").arg(arg); + } + if let Ok(arg) = test_env.read_json_object::(JAEGER_ADDR_PATH) { + cmd.arg("--jaeger_addr").arg(arg); + } + if let Ok(arg) = test_env.read_json_object::(SOCKS_PROXY_PATH) { + cmd.arg("--socks_proxy").arg(arg); + } + let key = "PATH"; let old_path = match std::env::var(key) { Ok(val) => { From dd8364ebc286a690472295f25b3aabfa1ec97e08 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 11 Dec 2024 21:52:37 +0000 Subject: [PATCH 222/241] Fix ipv4 integration test bug --- .../components/hostos-scripts/build-bootstrap-config-image.sh | 4 ---- rs/tests/driver/src/driver/bootstrap.rs | 3 --- 2 files changed, 7 deletions(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index 9fc449f7df4..a4998f48f1d 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -112,10 +112,6 @@ options may be specified: --socks_proxy url The URL of the socks proxy to use. To be used in systems tests only. - - --generate_ic_boundary_tls_cert domain_name - Generate and inject a self-signed TLS certificate and key for ic-boundary - for the given domain name. To be used in system tests only. EOF } diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 462c0682980..74eeffc9731 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -606,9 +606,6 @@ fn create_config_disk_image( )); cmd.arg("--ipv4_gateway").arg(ipv4_config.gateway_ip_addr()); } - if let Some(domain_name) = &node.node_config.domain { - cmd.arg("--generate_ic_boundary_tls_cert").arg(domain_name); - } if let Some(domain_name) = domain_name { cmd.arg("--domain").arg(domain_name); } From 23ff24c29a99cc0437afbc14966b60956d5a1940 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Mon, 23 Dec 2024 21:24:41 +0000 Subject: [PATCH 223/241] Use let-else expression --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 6 ++---- rs/ic_os/os_tools/setupos_tool/src/main.rs | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 4fb82b76333..236624970c0 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -85,11 +85,9 @@ pub fn main() -> Result<()> { eprintln!("Using generated mac address {}", generated_mac); - let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = + let Ipv6Config::Deterministic(ipv6_config) = &hostos_config.network_settings.ipv6_config - { - ipv6_config - } else { + else { return Err(anyhow!( "Ipv6Config is not of type Deterministic. Cannot generate IPv6 address." )); diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 4abef18d69b..2b471329e52 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -90,11 +90,9 @@ pub fn main() -> Result<()> { ); eprintln!("Using generated mac address {}", generated_mac); - let ipv6_config = if let Ipv6Config::Deterministic(ipv6_config) = + let Ipv6Config::Deterministic(ipv6_config) = &setupos_config.network_settings.ipv6_config - { - ipv6_config - } else { + else { return Err(anyhow!( "Ipv6Config is not of type Deterministic. Cannot generate IPv6 address." )); From 947c4196b074b095fb32de8c40d8361a0a6a7ba5 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 8 Jan 2025 16:34:36 +0000 Subject: [PATCH 224/241] Remove unnecessary config import --- ic-os/components/setupos-scripts/check-hardware.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ic-os/components/setupos-scripts/check-hardware.sh b/ic-os/components/setupos-scripts/check-hardware.sh index acf66ab5861..67261c63216 100644 --- a/ic-os/components/setupos-scripts/check-hardware.sh +++ b/ic-os/components/setupos-scripts/check-hardware.sh @@ -6,7 +6,6 @@ set -o pipefail SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" -source /opt/ic/bin/config.sh source /opt/ic/bin/functions.sh GENERATION= From d1307dd6046204145f3d71a69b67da040cb8eb48 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 8 Jan 2025 16:45:51 +0000 Subject: [PATCH 225/241] Remove unnecessary SetupOSArgs --- rs/ic_os/os_tools/setupos_tool/src/main.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 2b471329e52..35e6b9c67a5 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -3,10 +3,7 @@ use std::path::Path; use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; -use config::{ - deserialize_config, DEFAULT_SETUPOS_CONFIG_INI_FILE_PATH, DEFAULT_SETUPOS_CONFIG_OBJECT_PATH, - DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH, -}; +use config::{deserialize_config, DEFAULT_SETUPOS_CONFIG_OBJECT_PATH}; use config_types::{Ipv6Config, SetupOSConfig}; use deterministic_ips::node_type::NodeType; use deterministic_ips::{calculate_deterministic_mac, IpVariant, MacAddr6Ext}; @@ -30,13 +27,6 @@ pub enum Commands { #[derive(Parser)] struct SetupOSArgs { - #[arg(short, long, default_value_t = DEFAULT_SETUPOS_CONFIG_INI_FILE_PATH.to_string(), value_name = "FILE")] - config: String, - - #[arg(short, long, default_value_t = DEFAULT_SETUPOS_DEPLOYMENT_JSON_PATH.to_string(), value_name = "FILE")] - /// deployment.json file path - deployment_file: String, - #[command(subcommand)] command: Option, } From d087d074c3bbabb20132f8063ed2d4baef33459e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 8 Jan 2025 16:52:24 +0000 Subject: [PATCH 226/241] Parameterize setupos_config_object_path --- rs/ic_os/os_tools/setupos_tool/src/main.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rs/ic_os/os_tools/setupos_tool/src/main.rs b/rs/ic_os/os_tools/setupos_tool/src/main.rs index 35e6b9c67a5..2e2974ed31a 100644 --- a/rs/ic_os/os_tools/setupos_tool/src/main.rs +++ b/rs/ic_os/os_tools/setupos_tool/src/main.rs @@ -27,6 +27,9 @@ pub enum Commands { #[derive(Parser)] struct SetupOSArgs { + #[arg(short, long, default_value_t = DEFAULT_SETUPOS_CONFIG_OBJECT_PATH.to_string(), value_name = "FILE")] + setupos_config_object_path: String, + #[command(subcommand)] command: Option, } @@ -42,7 +45,7 @@ pub fn main() -> Result<()> { match opts.command { Some(Commands::GenerateNetworkConfig { output_directory }) => { let setupos_config: SetupOSConfig = - deserialize_config(DEFAULT_SETUPOS_CONFIG_OBJECT_PATH)?; + deserialize_config(&opts.setupos_config_object_path)?; eprintln!( "Network settings config: {:?}", @@ -65,7 +68,7 @@ pub fn main() -> Result<()> { } Some(Commands::GenerateIpv6Address { node_type }) => { let setupos_config: SetupOSConfig = - deserialize_config(DEFAULT_SETUPOS_CONFIG_OBJECT_PATH)?; + deserialize_config(&opts.setupos_config_object_path)?; eprintln!( "Network settings config: {:?}", From b1492805c26a1fdbd68d2d1c4c421c113a75af2c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 8 Jan 2025 16:55:08 +0000 Subject: [PATCH 227/241] Parameterize hostos_config_object_path --- rs/ic_os/os_tools/hostos_tool/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rs/ic_os/os_tools/hostos_tool/src/main.rs b/rs/ic_os/os_tools/hostos_tool/src/main.rs index 236624970c0..934af114904 100644 --- a/rs/ic_os/os_tools/hostos_tool/src/main.rs +++ b/rs/ic_os/os_tools/hostos_tool/src/main.rs @@ -31,6 +31,9 @@ pub enum Commands { #[derive(Parser)] struct HostOSArgs { + #[arg(short, long, default_value_t = DEFAULT_HOSTOS_CONFIG_OBJECT_PATH.to_string(), value_name = "FILE")] + hostos_config_object_path: String, + #[command(subcommand)] command: Option, } @@ -46,8 +49,7 @@ pub fn main() -> Result<()> { match opts.command { Some(Commands::GenerateNetworkConfig { output_directory }) => { - let hostos_config: HostOSConfig = - deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + let hostos_config: HostOSConfig = deserialize_config(&opts.hostos_config_object_path)?; eprintln!( "Network settings config: {:?}", @@ -68,8 +70,7 @@ pub fn main() -> Result<()> { ) } Some(Commands::GenerateIpv6Address { node_type }) => { - let hostos_config: HostOSConfig = - deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + let hostos_config: HostOSConfig = deserialize_config(&opts.hostos_config_object_path)?; eprintln!( "Network settings config: {:?}", @@ -99,8 +100,7 @@ pub fn main() -> Result<()> { Ok(()) } Some(Commands::GenerateMacAddress { node_type }) => { - let hostos_config: HostOSConfig = - deserialize_config(DEFAULT_HOSTOS_CONFIG_OBJECT_PATH)?; + let hostos_config: HostOSConfig = deserialize_config(&opts.hostos_config_object_path)?; eprintln!( "Network settings config: {:?}", From cfb7fadb64f8ea3401766d033f1ce8df20413eb3 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 8 Jan 2025 21:45:49 +0000 Subject: [PATCH 228/241] Update cargo.lock --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c90f884c3a3..24616cc7162 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12630,7 +12630,7 @@ dependencies = [ "candid", "canister-test", "chrono", - "clap 4.5.20", + "clap 4.5.23", "config", "config_types", "crossbeam-channel", @@ -14828,7 +14828,7 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" name = "launch-single-vm" version = "0.1.0" dependencies = [ - "clap 4.5.20", + "clap 4.5.23", "config", "config_types", "ic-prep", From 7dcbd7750cafab6313fb779c3c7de2538b0a4c5f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Wed, 8 Jan 2025 22:36:16 +0000 Subject: [PATCH 229/241] Fix pre-commit --- ic-os/components/ic/generate-ic-config/generate-ic-config.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh index 5ba35ba3218..180c7d83124 100755 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh @@ -99,7 +99,6 @@ function get_if_address_retries() { done } - function read_config_variables() { NNS_URLS=$(get_config_value '.icos_settings.nns_urls | join(",")') BACKUP_RETENTION_TIME_SECS=$(get_config_value '.guestos_settings.guestos_dev_settings.backup_spool.backup_retention_time_seconds') From 8f5625660fcb70089039471607d4a5eac64ed6ca Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 19:11:12 +0000 Subject: [PATCH 230/241] Remove unnecessary TODO --- ic-os/components/hostos-scripts/build-bootstrap-config-image.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh index a4998f48f1d..3fdeeb6fad0 100755 --- a/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh +++ b/ic-os/components/hostos-scripts/build-bootstrap-config-image.sh @@ -249,7 +249,6 @@ function build_ic_bootstrap_tar() { cp -r "${IC_REGISTRY_LOCAL_STORE}" "${BOOTSTRAP_TMPDIR}/ic_registry_local_store" fi - # TODO(NODE-1518): remove parsing for old config [[ "$HOSTNAME" == "" ]] || [[ "$HOSTNAME" =~ [a-zA-Z]*([a-zA-Z0-9])*(-+([a-zA-Z0-9])) ]] || { echo "Invalid hostname: '$HOSTNAME'" >&2 exit 1 From 65b96937b90c41fe855cf4031187920b7098ce3c Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 20:54:47 +0000 Subject: [PATCH 231/241] Revert log-config changes --- ic-os/components/guestos.bzl | 2 -- .../log-config/log-config.service | 4 +--- .../log-config/log-config.sh | 6 ++++-- ic-os/components/hostos.bzl | 4 ++-- .../misc/log-config/log-config-guestos.service | 14 -------------- 5 files changed, 7 insertions(+), 23 deletions(-) rename ic-os/components/{misc => hostos-scripts}/log-config/log-config.service (58%) rename ic-os/components/{misc => hostos-scripts}/log-config/log-config.sh (82%) delete mode 100644 ic-os/components/misc/log-config/log-config-guestos.service diff --git a/ic-os/components/guestos.bzl b/ic-os/components/guestos.bzl index bbd1d545d07..7c3fc59a284 100644 --- a/ic-os/components/guestos.bzl +++ b/ic-os/components/guestos.bzl @@ -63,8 +63,6 @@ component_files = { Label("misc/guestos/sysctl.d/dfn-max-map-count.conf"): "/etc/sysctl.d/dfn-max-map-count.conf", Label("misc/guestos/sysctl.d/privileged-ports.conf"): "/etc/sysctl.d/privileged-ports.conf", Label("misc/guestos/sysfs.d/hugepage.conf"): "/etc/sysfs.d/hugepage.conf", - Label("misc/log-config/log-config-guestos.service"): "/etc/systemd/system/log-config.service", - Label("misc/log-config/log-config.sh"): "/opt/ic/bin/log-config.sh", Label("misc/guestos/hsm/pcscd"): "/etc/default/pcscd", # monitoring diff --git a/ic-os/components/misc/log-config/log-config.service b/ic-os/components/hostos-scripts/log-config/log-config.service similarity index 58% rename from ic-os/components/misc/log-config/log-config.service rename to ic-os/components/hostos-scripts/log-config/log-config.service index 90bb67bbe41..b5e319f5a6f 100644 --- a/ic-os/components/misc/log-config/log-config.service +++ b/ic-os/components/hostos-scripts/log-config/log-config.service @@ -1,7 +1,5 @@ [Unit] -Description=Log config partition -After=update-config.service -Wants=update-config.service +Description=Log HostOS config partition [Service] Type=oneshot diff --git a/ic-os/components/misc/log-config/log-config.sh b/ic-os/components/hostos-scripts/log-config/log-config.sh similarity index 82% rename from ic-os/components/misc/log-config/log-config.sh rename to ic-os/components/hostos-scripts/log-config/log-config.sh index afb4655e306..8173407bddb 100644 --- a/ic-os/components/misc/log-config/log-config.sh +++ b/ic-os/components/hostos-scripts/log-config/log-config.sh @@ -1,7 +1,8 @@ #!/bin/bash CONFIG_DIR="/boot/config" -CONFIG="/boot/config/config.json" +CONFIG="/boot/config/config.ini" +DEPLOYMENT="/boot/config/deployment.json" log_directory_structure() { local dir=$1 @@ -27,6 +28,7 @@ log_file_contents() { fi } -echo "Logging config partition" +echo "Logging HostOS config partition" log_directory_structure "$CONFIG_DIR" log_file_contents "$CONFIG" +log_file_contents "$DEPLOYMENT" \ No newline at end of file diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 6940cdf7c9d..c5989243789 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -29,6 +29,8 @@ component_files = { Label("hostos-scripts/verbose-logging/verbose-logging.sh"): "/opt/ic/bin/verbose-logging.sh", Label("hostos-scripts/verbose-logging/verbose-logging.service"): "/etc/systemd/system/verbose-logging.service", Label("hostos-scripts/verbose-logging/logrotate.d/verbose-logging"): "/etc/logrotate.d/verbose-logging", + Label("hostos-scripts/log-config/log-config.service"): "/etc/systemd/system/log-config.service", + Label("hostos-scripts/log-config/log-config.sh"): "/opt/ic/bin/log-config.sh", # early-boot Label("early-boot/relabel-machine-id/relabel-machine-id.sh"): "/opt/ic/bin/relabel-machine-id.sh", @@ -57,8 +59,6 @@ component_files = { Label("misc/hostos/sudoers"): "/etc/sudoers", Label("misc/hostos/ic-node.conf"): "/etc/tmpfiles.d/ic-node.conf", Label("misc/hostos/20-ipmi.rules"): "/etc/udev/rules.d/20-ipmi.rules", - Label("misc/log-config/log-config.service"): "/etc/systemd/system/log-config.service", - Label("misc/log-config/log-config.sh"): "/opt/ic/bin/log-config.sh", # monitoring Label("monitoring/systemd-user/user@.service"): "/etc/systemd/system/user@.service", diff --git a/ic-os/components/misc/log-config/log-config-guestos.service b/ic-os/components/misc/log-config/log-config-guestos.service deleted file mode 100644 index 705bdfb511c..00000000000 --- a/ic-os/components/misc/log-config/log-config-guestos.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Log config partition -After=bootstrap-ic-node.service -Requires=bootstrap-ic-node.service -After=update-config.service -Wants=update-config.service - -[Service] -Type=oneshot -ExecStart=/opt/ic/bin/log-config.sh -RemainAfterExit=true - -[Install] -WantedBy=multi-user.target \ No newline at end of file From 02ca2455503c4e2081f14d78e2c44b75c0339908 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 20:55:22 +0000 Subject: [PATCH 232/241] Add newline to log-config.sh --- ic-os/components/hostos-scripts/log-config/log-config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/hostos-scripts/log-config/log-config.sh b/ic-os/components/hostos-scripts/log-config/log-config.sh index 8173407bddb..5f8399c90b4 100644 --- a/ic-os/components/hostos-scripts/log-config/log-config.sh +++ b/ic-os/components/hostos-scripts/log-config/log-config.sh @@ -31,4 +31,4 @@ log_file_contents() { echo "Logging HostOS config partition" log_directory_structure "$CONFIG_DIR" log_file_contents "$CONFIG" -log_file_contents "$DEPLOYMENT" \ No newline at end of file +log_file_contents "$DEPLOYMENT" From 3d7d57c5599428c4029effa5fa429171e9a65258 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 21:01:42 +0000 Subject: [PATCH 233/241] Separate hostos and guestos setup-hostname --- .../{ => guestos}/setup-hostname.service | 0 .../setup-hostname/guestos/setup-hostname.sh | 107 ++++++++++++++++++ .../{ => hostos}/setup-hostname.sh | 0 ic-os/components/guestos.bzl | 4 +- ic-os/components/hostos.bzl | 2 +- 5 files changed, 110 insertions(+), 3 deletions(-) rename ic-os/components/early-boot/setup-hostname/{ => guestos}/setup-hostname.service (100%) create mode 100644 ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh rename ic-os/components/early-boot/setup-hostname/{ => hostos}/setup-hostname.sh (100%) diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.service b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.service similarity index 100% rename from ic-os/components/early-boot/setup-hostname/setup-hostname.service rename to ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.service diff --git a/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh new file mode 100644 index 00000000000..d84093f8ae1 --- /dev/null +++ b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +set -e + +# Set the transient or persistent hostname. + +source /opt/ic/bin/logging.sh +source /opt/ic/bin/metrics.sh + +SCRIPT="$(basename $0)[$$]" + +# Get keyword arguments +for argument in "${@}"; do + case ${argument} in + -c=* | --config=*) + CONFIG="${argument#*=}" + shift + ;; + -f=* | --file=*) + FILE="${argument#*=}" + shift + ;; + -h | --help) + echo 'Usage: +Set Transient Or Persistent Hostname +Arguments: + -c=, --config= optional: specify the config.ini configuration file (Default: /boot/config/config.ini) + -f=, --file= optional: specify the file containing the node-id (Default: /boot/config/node-id) + -h, --help show this help message and exit + -t=, --type= mandatory: specify the node type (Examples: host, guest, boundary...) +' + exit 1 + ;; + -t=* | --type=*) + TYPE="${argument#*=}" + shift + ;; + *) + echo "Error: Argument is not supported." + exit 1 + ;; + esac +done + +# Set arguments if undefined +CONFIG="${CONFIG:=/boot/config/config.ini}" +FILE="${FILE:=/boot/config/node-id}" + +function validate_arguments() { + if [ "${CONFIG}" == "" -o "${FILE}" == "" -o "${TYPE}" == "" ]; then + $0 --help + fi +} + +function read_variables() { + # Read limited set of keys. Be extra-careful quoting values as it could + # otherwise lead to executing arbitrary shell code! + while IFS="=" read -r key value; do + case "$key" in + "ipv6_prefix") ipv6_prefix="${value}" ;; + "ipv6_gateway") ipv6_gateway="${value}" ;; + esac + done <"${CONFIG}" +} + +function construct_hostname() { + local mac=$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g') + + if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then + HOSTNAME=$(echo ${TYPE}-${mac}-$(cat ${FILE})) + write_log "Using hostname: ${HOSTNAME}" + write_metric "hostos_setup_hostname" \ + "1" \ + "HostOS setup hostname" \ + "gauge" + else + HOSTNAME=$(echo ${TYPE}-${mac}) + write_log "Using hostname: ${HOSTNAME}" + write_metric "hostos_setup_hostname" \ + "0" \ + "HostOS setup hostname" \ + "gauge" + fi +} + +function setup_hostname() { + if [ "$(mount | grep '/etc/hostname')" ]; then + umount /etc/hostname + fi + + if [ -d /run/ic-node/etc ]; then + echo "${HOSTNAME}" >/run/ic-node/etc/hostname + mount --bind /run/ic-node/etc/hostname /etc/hostname + restorecon -v /etc/hostname + hostname "${HOSTNAME}" + fi +} + +function main() { + # Establish run order + validate_arguments + read_variables + construct_hostname + setup_hostname +} + +main diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh similarity index 100% rename from ic-os/components/early-boot/setup-hostname/setup-hostname.sh rename to ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh diff --git a/ic-os/components/guestos.bzl b/ic-os/components/guestos.bzl index 7c3fc59a284..299ff8a12a5 100644 --- a/ic-os/components/guestos.bzl +++ b/ic-os/components/guestos.bzl @@ -6,8 +6,8 @@ component_files = { # early-boot Label("early-boot/relabel-machine-id/guestos/relabel-machine-id.sh"): "/opt/ic/bin/relabel-machine-id.sh", Label("early-boot/relabel-machine-id/relabel-machine-id.service"): "/etc/systemd/system/relabel-machine-id.service", - Label("early-boot/setup-hostname/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", - Label("early-boot/setup-hostname/setup-hostname.service"): "/etc/systemd/system/setup-hostname.service", + Label("early-boot/setup-hostname/guestos/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", + Label("early-boot/setup-hostname/guestos/setup-hostname.service"): "/etc/systemd/system/setup-hostname.service", Label("early-boot/setup-hostname/hostname-empty"): "/etc/hostname", Label("early-boot/save-machine-id/save-machine-id.sh"): "/opt/ic/bin/save-machine-id.sh", Label("early-boot/save-machine-id/save-machine-id.service"): "/etc/systemd/system/save-machine-id.service", diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index c5989243789..f1c219868ea 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -35,7 +35,7 @@ component_files = { # early-boot Label("early-boot/relabel-machine-id/relabel-machine-id.sh"): "/opt/ic/bin/relabel-machine-id.sh", Label("early-boot/relabel-machine-id/relabel-machine-id.service"): "/etc/systemd/system/relabel-machine-id.service", - Label("early-boot/setup-hostname/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", + Label("early-boot/setup-hostname/hostos/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", Label("early-boot/setup-hostname/hostos/setup-hostname.service"): "/etc/systemd/system/setup-hostname.service", Label("early-boot/setup-hostname/hostname-empty"): "/etc/hostname", Label("early-boot/save-machine-id/save-machine-id.sh"): "/opt/ic/bin/save-machine-id.sh", From caf320b2d4aa62c9977ff80f4b68bbf3cda978b0 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 21:07:36 +0000 Subject: [PATCH 234/241] Move hostos setup-hostname.sh up --- .../early-boot/setup-hostname/{hostos => }/setup-hostname.sh | 0 ic-os/components/hostos.bzl | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename ic-os/components/early-boot/setup-hostname/{hostos => }/setup-hostname.sh (100%) diff --git a/ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh similarity index 100% rename from ic-os/components/early-boot/setup-hostname/hostos/setup-hostname.sh rename to ic-os/components/early-boot/setup-hostname/setup-hostname.sh diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index f1c219868ea..c5989243789 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -35,7 +35,7 @@ component_files = { # early-boot Label("early-boot/relabel-machine-id/relabel-machine-id.sh"): "/opt/ic/bin/relabel-machine-id.sh", Label("early-boot/relabel-machine-id/relabel-machine-id.service"): "/etc/systemd/system/relabel-machine-id.service", - Label("early-boot/setup-hostname/hostos/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", + Label("early-boot/setup-hostname/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh", Label("early-boot/setup-hostname/hostos/setup-hostname.service"): "/etc/systemd/system/setup-hostname.service", Label("early-boot/setup-hostname/hostname-empty"): "/etc/hostname", Label("early-boot/save-machine-id/save-machine-id.sh"): "/opt/ic/bin/save-machine-id.sh", From 9efc624449f0e2233f222177923e40911486fc58 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 21:09:00 +0000 Subject: [PATCH 235/241] Add newline to setup-hostname.sh guestos --- .../early-boot/setup-hostname/guestos/setup-hostname.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh index d84093f8ae1..088b3f93e76 100644 --- a/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh @@ -23,6 +23,7 @@ for argument in "${@}"; do -h | --help) echo 'Usage: Set Transient Or Persistent Hostname + Arguments: -c=, --config= optional: specify the config.ini configuration file (Default: /boot/config/config.ini) -f=, --file= optional: specify the file containing the node-id (Default: /boot/config/node-id) From ab57f9e6150c79a0843f8dac29463661e3b5f1b3 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 21:10:21 +0000 Subject: [PATCH 236/241] Fix setup-hostname.sh guestos --- .../setup-hostname/guestos/setup-hostname.sh | 120 ++++-------------- 1 file changed, 22 insertions(+), 98 deletions(-) diff --git a/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh index 088b3f93e76..1d794bace26 100644 --- a/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.sh @@ -1,108 +1,32 @@ #!/bin/bash -set -e - -# Set the transient or persistent hostname. - -source /opt/ic/bin/logging.sh -source /opt/ic/bin/metrics.sh - -SCRIPT="$(basename $0)[$$]" - -# Get keyword arguments -for argument in "${@}"; do - case ${argument} in - -c=* | --config=*) - CONFIG="${argument#*=}" - shift - ;; - -f=* | --file=*) - FILE="${argument#*=}" - shift - ;; - -h | --help) - echo 'Usage: -Set Transient Or Persistent Hostname - -Arguments: - -c=, --config= optional: specify the config.ini configuration file (Default: /boot/config/config.ini) - -f=, --file= optional: specify the file containing the node-id (Default: /boot/config/node-id) - -h, --help show this help message and exit - -t=, --type= mandatory: specify the node type (Examples: host, guest, boundary...) -' - exit 1 - ;; - -t=* | --type=*) - TYPE="${argument#*=}" - shift - ;; - *) - echo "Error: Argument is not supported." - exit 1 - ;; - esac -done - -# Set arguments if undefined -CONFIG="${CONFIG:=/boot/config/config.ini}" -FILE="${FILE:=/boot/config/node-id}" - -function validate_arguments() { - if [ "${CONFIG}" == "" -o "${FILE}" == "" -o "${TYPE}" == "" ]; then - $0 --help - fi -} - +set -ex + +# Read the network config variables from file. The file must be of the form +# "key=value" for each line with a specific set of keys permissible (see +# code below). +# +# Arguments: +# - $1: Name of the file to be read. function read_variables() { # Read limited set of keys. Be extra-careful quoting values as it could # otherwise lead to executing arbitrary shell code! while IFS="=" read -r key value; do case "$key" in - "ipv6_prefix") ipv6_prefix="${value}" ;; - "ipv6_gateway") ipv6_gateway="${value}" ;; + "hostname") hostname="${value}" ;; esac - done <"${CONFIG}" -} - -function construct_hostname() { - local mac=$(/opt/ic/bin/hostos_tool fetch-mac-address | sed 's/://g') - - if [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then - HOSTNAME=$(echo ${TYPE}-${mac}-$(cat ${FILE})) - write_log "Using hostname: ${HOSTNAME}" - write_metric "hostos_setup_hostname" \ - "1" \ - "HostOS setup hostname" \ - "gauge" - else - HOSTNAME=$(echo ${TYPE}-${mac}) - write_log "Using hostname: ${HOSTNAME}" - write_metric "hostos_setup_hostname" \ - "0" \ - "HostOS setup hostname" \ - "gauge" - fi -} - -function setup_hostname() { - if [ "$(mount | grep '/etc/hostname')" ]; then - umount /etc/hostname - fi - - if [ -d /run/ic-node/etc ]; then - echo "${HOSTNAME}" >/run/ic-node/etc/hostname - mount --bind /run/ic-node/etc/hostname /etc/hostname - restorecon -v /etc/hostname - hostname "${HOSTNAME}" - fi -} - -function main() { - # Establish run order - validate_arguments - read_variables - construct_hostname - setup_hostname + done <"$1" } -main +if [ -e /boot/config/network.conf ]; then + cat /boot/config/network.conf + read_variables /boot/config/network.conf + hostname="${hostname:-blank}" +else + hostname="unnamed" +fi + +echo "${hostname}" >/run/ic-node/etc/hostname +mount --bind /run/ic-node/etc/hostname /etc/hostname +restorecon -v /etc/hostname +hostname "${hostname}" From 41fa04a13c2ba27585ec9b69734b7b04711c16ff Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 9 Jan 2025 21:37:35 +0000 Subject: [PATCH 237/241] Fix setup-hostname --- .../early-boot/setup-hostname/guestos/setup-hostname.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.service b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.service index 7655e284632..c2f8b7352ff 100644 --- a/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.service +++ b/ic-os/components/early-boot/setup-hostname/guestos/setup-hostname.service @@ -12,4 +12,4 @@ WantedBy=multi-user.target [Service] Type=oneshot RemainAfterExit=true -ExecStart=/opt/ic/bin/setup-hostname.sh --type=guest +ExecStart=/opt/ic/bin/setup-hostname.sh From 049cfe10dc0987d9110cdcc283e3f46fd07d80cb Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Fri, 10 Jan 2025 18:40:51 +0000 Subject: [PATCH 238/241] Fix generate-guestos-config --- .../generate-guestos-config/dev-generate-guestos-config.sh | 5 ++++- .../generate-guestos-config/generate-guestos-config.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 694bb13aeba..93496b6661d 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -88,7 +88,10 @@ function assemble_config_media() { fi cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then + if [[ -n "$ipv4_address" && "$ipv4_address" != "null" \ + && -n "$ipv4_prefix_length" && "$ipv4_prefix_length" != "null" \ + && -n "$ipv4_gateway" && "$ipv4_gateway" != "null" \ + && -n "$domain_name" && "$domain_name" != "null" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain_name}") diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 89ea53fa8b4..fb150311eaf 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -84,7 +84,10 @@ function assemble_config_media() { fi cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then + if [[ -n "$ipv4_address" && "$ipv4_address" != "null" \ + && -n "$ipv4_prefix_length" && "$ipv4_prefix_length" != "null" \ + && -n "$ipv4_gateway" && "$ipv4_gateway" != "null" \ + && -n "$domain_name" && "$domain_name" != "null" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain_name}") From 722b30931e03fc054ed60814d8071ae6396a528e Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 28 Jan 2025 20:06:57 +0000 Subject: [PATCH 239/241] Normalize config.sh get_config_value --- Cargo.lock | 1221 ++++++++--------- .../setup-hostname/setup-hostname.sh | 2 +- .../dev-generate-guestos-config.sh | 5 +- .../generate-guestos-config.sh | 5 +- ic-os/components/misc/config/config.sh | 13 +- 5 files changed, 587 insertions(+), 659 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e0457b02a2..ea06f55e2b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,7 +39,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "bytes", "futures-core", "futures-sink", @@ -62,7 +62,7 @@ dependencies = [ "actix-utils", "ahash 0.8.11", "base64 0.22.1", - "bitflags 2.8.0", + "bitflags 2.6.0", "brotli 6.0.0", "bytes", "bytestring", @@ -96,7 +96,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -214,7 +214,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -292,15 +292,15 @@ dependencies = [ [[package]] name = "aide" -version = "0.13.5" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5678d2978845ddb4bd736a026f467dd652d831e9e6254b0e41b07f7ee7523309" +checksum = "7b0e3b97a21e41ec5c19bfd9b4fc1f7086be104f8b988681230247ffc91cc8ed" dependencies = [ "axum", "bytes", "cfg-if 1.0.0", "http 1.2.0", - "indexmap 2.7.1", + "indexmap 2.7.0", "schemars", "serde", "serde_json", @@ -375,7 +375,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "rsa", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", ] @@ -403,7 +403,7 @@ dependencies = [ "lazy_static", "prometheus", "serde", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -447,20 +447,19 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "once_cell", "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arbitrary" @@ -526,7 +525,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -568,7 +567,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", "synstructure", ] @@ -580,7 +579,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -692,7 +691,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 5.4.0", + "event-listener 5.3.1", "event-listener-strategy", "pin-project-lite", ] @@ -748,7 +747,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -759,13 +758,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.85" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -814,13 +813,13 @@ dependencies = [ [[package]] name = "auto_impl" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12882f59de5360c748c4cbf569a042d5fb0eb515f7bea9c1f470b47f6ffbd73" +checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -1109,7 +1108,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -1120,7 +1119,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -1340,9 +1339,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bitvec" @@ -1389,9 +1388,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.5.5" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5430e3be710b68d984d1391c854eb431a9d548640711faa54eecb1df93db91cc" +checksum = "2506947f73ad44e344215ccd6403ac2ae18cd8e046e581a441bf8d199f257f03" dependencies = [ "borsh-derive", "cfg_aliases", @@ -1399,15 +1398,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.5" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b668d39970baad5356d7c83a86fee3a539e6f93bf6764c97368243e17a0487" +checksum = "c2593a3b8b938bd68373196c9832f516be11fa487ef4ae745eb282e6a56a7244" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -1434,9 +1433,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.2" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1454,9 +1453,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" dependencies = [ "memchr", "regex-automata 0.4.9", @@ -1577,9 +1576,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.21.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -1677,7 +1676,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -1713,7 +1712,7 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", - "clap 4.5.27", + "clap 4.5.23", "futures-util", "http 1.2.0", "http-body 1.0.1", @@ -1753,9 +1752,9 @@ dependencies = [ [[package]] name = "candid" -version = "0.10.12" +version = "0.10.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e129c4051c57daf943586e01ef72faae48b04a8f692d5f646febf17a264c38" +checksum = "a253bab4a9be502c82332b60cbeee6202ad0692834efeec95fae9f29db33d692" dependencies = [ "anyhow", "binread", @@ -1792,7 +1791,7 @@ dependencies = [ "lazy_static", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -1900,9 +1899,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.10" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" dependencies = [ "jobserver", "libc", @@ -1952,7 +1951,7 @@ dependencies = [ "candid", "certificate_orchestrator_interface", "chacha20poly1305", - "clap 4.5.27", + "clap 4.5.23", "cloudflare 0.12.0 (git+https://github.com/dfinity/cloudflare-rs.git?rev=a6538a036926bd756986c9c0a5de356daef48881)", "flate2", "futures", @@ -1960,7 +1959,7 @@ dependencies = [ "ic-agent", "ic-http-certification", "ic-response-verification", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "idna 1.0.3", "instant-acme", "leb128", @@ -1970,12 +1969,12 @@ dependencies = [ "pem 1.1.1", "prometheus", "rcgen", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_cbor", "serde_json", "sha2 0.10.8", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tower 0.5.2", "tracing", @@ -2007,7 +2006,7 @@ dependencies = [ "serde", "serde_cbor", "sha2 0.10.8", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -2019,7 +2018,7 @@ dependencies = [ "ic-stable-structures", "serde", "serde_bytes", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -2162,19 +2161,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.27" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", - "clap_derive 4.5.24", + "clap_derive 4.5.18", ] [[package]] name = "clap_builder" -version = "4.5.27" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -2197,14 +2196,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -2380,7 +2379,7 @@ name = "config" version = "1.0.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "config_types", "ic-types", "macaddr", @@ -2407,7 +2406,7 @@ dependencies = [ "serde_json", "serde_with 1.14.0", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "url", ] @@ -2516,7 +2515,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_json", "slog", "tokio", @@ -2720,18 +2719,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88c1d02b72b6c411c0a2e92b25ed791ad5d071184193c08a34aa0fdcdf000b72" +checksum = "ac89549be94911dd0e839b4a7db99e9ed29c17517e1c026f61066884c168aa3c" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "720b93bd86ebbb23ebfb2db1ed44d54b2ecbdbb2d034d485bc64aa605ee787ab" +checksum = "b9bd49369f76c77e34e641af85d0956869237832c118964d08bf5f51f210875a" dependencies = [ "serde", "serde_derive", @@ -2739,9 +2738,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aed3d2d9914d30b460eedd7fd507720203023997bef71452ce84873f9c93537c" +checksum = "fd96ce9cf8efebd7f5ab8ced5a0ce44250280bbae9f593d74a6d7effc3582a35" dependencies = [ "bumpalo", "cranelift-bforest", @@ -2763,33 +2762,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "888c188d32263ec9e048873ff0b68c700933600d553f4412417916828be25f8e" +checksum = "5a68e358827afe4bfb6239fcbf6fbd5ac56206ece8a99c8f5f9bbd518773281a" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddd5f4114d04ce7e073dd74e2ad16541fc61970726fcc8b2d5644a154ee4127" +checksum = "e184c9767afbe73d50c55ec29abcf4c32f9baf0d9d22b86d58c4d55e06dee181" [[package]] name = "cranelift-control" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92cc4c98d6a4256a1600d93ccd3536f3e77da9b4ca2c279de786ac22876e67d6" +checksum = "5cc7664f2a66f053e33f149e952bb5971d138e3af637f5097727ed6dc0ed95dd" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760af4b5e051b5f82097a27274b917e3751736369fa73660513488248d27f23d" +checksum = "118597e3a9cf86c3556fa579a7a23b955fa18231651a52a77a2475d305a9cf84" dependencies = [ "cranelift-bitset", "serde", @@ -2798,9 +2797,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0bf77ec0f470621655ec7539860b5c620d4f91326654ab21b075b83900f8831" +checksum = "7638ea1efb069a0aa18d8ee67401b6b0d19f6bfe5de5e9ede348bfc80bb0d8c7" dependencies = [ "cranelift-codegen", "log", @@ -2810,15 +2809,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b665d0a6932c421620be184f9fc7f7adaf1b0bc2fa77bb7ac5177c49abf645b" +checksum = "15c53e1152a0b01c4ed2b1e0535602b8e86458777dd9d18b28732b16325c7dc0" [[package]] name = "cranelift-native" -version = "0.115.1" +version = "0.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2e75d1bd43dfec10924798f15e6474f1dbf63b0024506551aa19394dbe72ab" +checksum = "7b7d8f895444fa52dd7bdd0bed11bf007a7fb43af65a6deac8fcc4094c6372f7" dependencies = [ "cranelift-codegen", "libc", @@ -2843,7 +2842,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.27", + "clap 4.5.23", "criterion-plot", "futures", "is-terminal", @@ -2942,7 +2941,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "crossterm_winapi", "libc", "mio 0.8.11", @@ -2999,7 +2998,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf 0.11.3", + "phf 0.11.2", "smallvec", ] @@ -3010,7 +3009,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3070,7 +3069,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3209,7 +3208,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3231,7 +3230,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3269,9 +3268,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.7.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "debugid" @@ -3324,7 +3323,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3345,7 +3344,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3356,7 +3355,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3369,7 +3368,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3389,7 +3388,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3397,11 +3396,11 @@ name = "deterministic_ips" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "config_types", "ic-crypto-sha2", "macaddr", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -3409,7 +3408,7 @@ name = "dflate" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "libc", "tar", ] @@ -3524,7 +3523,7 @@ name = "diroid" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "walkdir", ] @@ -3557,7 +3556,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3802,7 +3801,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3815,7 +3814,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -3865,7 +3864,7 @@ checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -4018,9 +4017,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.4.0" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", @@ -4033,7 +4032,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ - "event-listener 5.4.0", + "event-listener 5.3.1", "pin-project-lite", ] @@ -4048,7 +4047,7 @@ dependencies = [ "ic-cdk 0.16.0", "mockall", "serde", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", ] @@ -4131,11 +4130,11 @@ dependencies = [ "ic-types", "ic-types-test-utils", "ic-universal-canister", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "lazy_static", "rand 0.8.5", "rand_chacha 0.3.1", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_cbor", "slog", "tokio", @@ -4419,9 +4418,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.6.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" dependencies = [ "fastrand", "futures-core", @@ -4438,7 +4437,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -4448,7 +4447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-pki-types", ] @@ -4503,19 +4502,6 @@ version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" -[[package]] -name = "generator" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "log", - "rustversion", - "windows", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -4589,15 +4575,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" dependencies = [ "fallible-iterator 0.3.0", - "indexmap 2.7.1", + "indexmap 2.7.0", "stable_deref_trait", ] [[package]] name = "glob" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "governor" @@ -4635,7 +4621,7 @@ name = "guestos_tool" version = "1.0.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "config", "config_types", "indoc", @@ -4657,7 +4643,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.7.1", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -4676,7 +4662,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.2.0", - "indexmap 2.7.1", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -4973,7 +4959,7 @@ name = "hostos_tool" version = "1.0.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "config", "config_types", "deterministic_ips", @@ -5073,10 +5059,10 @@ name = "httpbin-rs" version = "0.9.0" dependencies = [ "axum", - "clap 4.5.27", + "clap 4.5.23", "hyper 1.5.2", "hyper-util", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "serde_json", "tokio", @@ -5205,7 +5191,7 @@ dependencies = [ "hyper 1.5.2", "hyper-util", "log", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -5272,7 +5258,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -5333,7 +5319,7 @@ dependencies = [ "base64 0.13.1", "candid", "chrono", - "clap 4.5.27", + "clap 4.5.23", "cycles-minting-canister", "futures", "hex", @@ -5377,7 +5363,7 @@ dependencies = [ "ic-sns-swap", "ic-sns-wasm", "ic-types", - "indexmap 2.7.1", + "indexmap 2.7.0", "itertools 0.12.1", "maplit", "pocket-ic", @@ -5403,9 +5389,9 @@ dependencies = [ [[package]] name = "ic-agent" -version = "0.39.3" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820d65a05258f2fdff326c65561b1ddc7ec54e5d43a4b1203b25eb83075c83d4" +checksum = "1ba408987ca48fc3eee6a613e760d076a9046cccbbb5ba29efbada339ab28ed9" dependencies = [ "arc-swap", "async-channel 1.9.0", @@ -5433,7 +5419,7 @@ dependencies = [ "pkcs8", "rand 0.8.5", "rangemap", - "reqwest 0.12.12", + "reqwest 0.12.9", "ring 0.17.8", "sec1", "serde", @@ -5443,7 +5429,7 @@ dependencies = [ "sha2 0.10.8", "simple_asn1", "stop-token", - "thiserror 2.0.11", + "thiserror 2.0.8", "time", "tokio", "tower-service", @@ -5480,7 +5466,7 @@ dependencies = [ "prost 0.13.4", "rand 0.8.5", "slog", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tower 0.5.2", "tracing", @@ -5512,7 +5498,7 @@ version = "0.9.0" dependencies = [ "bincode", "byteorder", - "clap 4.5.27", + "clap 4.5.23", "criterion", "ic-config", "ic-crypto-test-utils-canister-threshold-sigs", @@ -5555,7 +5541,7 @@ version = "0.9.0" dependencies = [ "anyhow", "chrono", - "clap 4.5.27", + "clap 4.5.23", "ic-config", "ic-crypto-utils-threshold-sig-der", "ic-logger", @@ -5567,7 +5553,7 @@ dependencies = [ "ic-test-utilities-tmpdir", "ic-types", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_json", "slog", @@ -5649,8 +5635,8 @@ dependencies = [ "base64 0.22.1", "bytes", "chacha20poly1305", - "clap 4.5.27", - "clap_derive 4.5.24", + "clap 4.5.23", + "clap_derive 4.5.18", "cloudflare 0.12.0 (git+https://github.com/cloudflare/cloudflare-rs.git?rev=f14720e42184ee176a97676e85ef2d2d85bc3aae)", "derive-new", "fqdn 0.4.4", @@ -5672,8 +5658,8 @@ dependencies = [ "prost-types 0.13.4", "rand 0.8.5", "rcgen", - "reqwest 0.12.12", - "rustls 0.23.21", + "reqwest 0.12.9", + "rustls 0.23.20", "rustls-acme", "rustls-pemfile 2.2.0", "rustls-platform-verifier", @@ -5684,7 +5670,7 @@ dependencies = [ "strum", "strum_macros", "systemstat", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tokio-io-timeout", "tokio-rustls 0.26.1", @@ -5713,7 +5699,7 @@ dependencies = [ "axum-extra", "bytes", "candid", - "clap 4.5.27", + "clap 4.5.23", "criterion", "dashmap 6.1.0", "ethnum", @@ -5765,8 +5751,8 @@ dependencies = [ "ratelimit", "rcgen", "regex", - "reqwest 0.12.12", - "rustls 0.23.21", + "reqwest 0.12.9", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "serde", "serde_bytes", @@ -5778,7 +5764,7 @@ dependencies = [ "slog", "strum", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", @@ -5815,7 +5801,7 @@ dependencies = [ "pem 1.1.1", "rand 0.8.5", "rand_chacha 0.3.1", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_json", "slog", "tokio", @@ -5832,7 +5818,7 @@ dependencies = [ "ic-crypto-tree-hash", "ic-system-test-driver", "ic-types", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_cbor", "slog", @@ -5856,7 +5842,7 @@ dependencies = [ "ic-system-test-driver", "prost 0.13.4", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "slog", "tokio", ] @@ -5877,7 +5863,7 @@ dependencies = [ "ic-registry-subnet-type", "ic-system-test-driver", "ic-types", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "slog", "url", ] @@ -5889,7 +5875,7 @@ dependencies = [ "bitcoin 0.32.5", "bitcoincore-rpc", "bitcoind", - "clap 4.5.27", + "clap 4.5.23", "criterion", "futures", "hashlink", @@ -5919,7 +5905,7 @@ dependencies = [ "slog", "slog-async", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tokio-socks", "tonic", @@ -6025,7 +6011,7 @@ dependencies = [ "proptest 1.6.0", "prost 0.13.4", "slog", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -6101,7 +6087,7 @@ dependencies = [ "prost 0.13.4", "rand 0.8.5", "rand_chacha 0.3.1", - "rustls 0.23.21", + "rustls 0.23.20", "serde", "serde_cbor", "tokio", @@ -6227,7 +6213,7 @@ dependencies = [ "serde_bytes", "serde_cbor", "sha2 0.10.8", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -6294,7 +6280,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "scoped_threadpool", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -6382,7 +6368,7 @@ dependencies = [ "ic0 0.24.0-alpha.1", "serde", "serde_bytes", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -6438,7 +6424,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream 0.2.2", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -6452,7 +6438,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream 0.2.2", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -6465,7 +6451,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream 0.2.2", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -6720,7 +6706,7 @@ dependencies = [ "strum", "strum_macros", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "thousands", "time", "tokio", @@ -6969,7 +6955,7 @@ dependencies = [ "assert_matches", "async-trait", "bincode", - "clap 4.5.27", + "clap 4.5.23", "criterion", "hex", "ic-adapter-metrics-server", @@ -7048,7 +7034,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rsa", - "rustls 0.23.21", + "rustls 0.23.20", "serde", "sha2 0.10.8", "simple_asn1", @@ -7092,7 +7078,7 @@ dependencies = [ "pem 1.1.1", "rand 0.8.5", "rand_chacha 0.3.1", - "thiserror 2.0.11", + "thiserror 2.0.8", "wycheproof", "zeroize", ] @@ -7398,7 +7384,7 @@ dependencies = [ "stubborn-io", "tarpc", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "time", "tokio", "tokio-serde", @@ -7640,7 +7626,7 @@ dependencies = [ "serde_cbor", "strum", "strum_macros", - "thiserror 2.0.11", + "thiserror 2.0.8", "zeroize", ] @@ -7809,7 +7795,7 @@ dependencies = [ "ic-types-test-utils", "rand 0.8.5", "rand_chacha 0.3.1", - "rustls 0.23.21", + "rustls 0.23.20", "tempfile", "tokio", ] @@ -7988,7 +7974,7 @@ version = "0.9.0" dependencies = [ "ic-types", "mockall", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -8009,7 +7995,7 @@ dependencies = [ "ic-types", "pkcs8", "rand 0.8.5", - "rustls 0.23.21", + "rustls 0.23.20", "signature", "time", "tokio", @@ -8045,9 +8031,9 @@ dependencies = [ "ic-types", "json5", "maplit", - "rustls 0.23.21", + "rustls 0.23.20", "serde", - "thiserror 2.0.11", + "thiserror 2.0.8", "x509-parser", ] @@ -8058,7 +8044,7 @@ dependencies = [ "ic-base-types", "ic-crypto-tls-interfaces", "mockall", - "rustls 0.23.21", + "rustls 0.23.20", ] [[package]] @@ -8079,7 +8065,7 @@ dependencies = [ "serde", "serde_bytes", "serde_cbor", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -8091,7 +8077,7 @@ dependencies = [ "ic-crypto-tree-hash", "proptest 1.6.0", "rand 0.8.5", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -8158,7 +8144,7 @@ name = "ic-crypto-utils-tls" version = "0.9.0" dependencies = [ "ic-base-types", - "thiserror 2.0.11", + "thiserror 2.0.8", "x509-parser", ] @@ -8173,7 +8159,7 @@ dependencies = [ "ic-registry-nns-data-provider", "ic-types", "prost 0.13.4", - "reqwest 0.12.12", + "reqwest 0.12.9", "tokio", ] @@ -8243,7 +8229,7 @@ dependencies = [ name = "ic-drun" version = "0.9.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "futures", "hex", "ic-canister-sandbox-backend-lib", @@ -8288,7 +8274,7 @@ dependencies = [ "bincode", "candid", "canister-test", - "clap 4.5.27", + "clap 4.5.23", "criterion", "embedders_bench", "ic-base-types", @@ -8467,7 +8453,7 @@ dependencies = [ "anyhow", "assert_cmd", "assert_matches", - "clap 4.5.27", + "clap 4.5.23", "ic-crypto-test-utils-reproducible-rng", "ic-sys", "maplit", @@ -8521,9 +8507,9 @@ dependencies = [ "ic-metrics", "ic-test-utilities-logger", "prometheus", - "reqwest 0.12.12", + "reqwest 0.12.9", "slog", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tokio-io-timeout", "tower 0.5.2", @@ -8586,7 +8572,7 @@ dependencies = [ "ic-tracing", "ic-types", "ic-validator", - "inferno 0.12.1", + "inferno 0.12.0", "maplit", "mockall", "pretty_assertions", @@ -8594,9 +8580,9 @@ dependencies = [ "proptest 1.6.0", "prost 0.13.4", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "rstest", - "rustls 0.23.21", + "rustls 0.23.20", "serde", "serde_bytes", "serde_cbor", @@ -8643,7 +8629,7 @@ dependencies = [ "maplit", "prometheus", "prost 0.13.4", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_json", "slog", @@ -8668,7 +8654,7 @@ dependencies = [ "ic-agent", "ic-http-certification", "ic-response-verification", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "thiserror 1.0.69", ] @@ -8684,7 +8670,7 @@ dependencies = [ "ic-logger", "ic-test-utilities-in-memory-logger", "mockito", - "reqwest 0.12.12", + "reqwest 0.12.9", "slog", "tar", "tempfile", @@ -8699,7 +8685,7 @@ dependencies = [ "async-stream", "byte-unit", "bytes", - "clap 4.5.27", + "clap 4.5.23", "futures", "http 1.2.0", "http-body-util", @@ -8718,13 +8704,13 @@ dependencies = [ "prometheus", "rand 0.8.5", "rstest", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "serde", "serde_json", "slog", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tokio-rustls 0.26.1", "tonic", @@ -8859,7 +8845,7 @@ dependencies = [ "icrc-ledger-types", "num-bigint 0.4.6", "pocket-ic", - "reqwest 0.12.12", + "reqwest 0.12.9", "rosetta-core", "serde", "tempfile", @@ -8876,7 +8862,7 @@ dependencies = [ "ic-rosetta-test-utils", "icp-ledger", "pocket-ic", - "reqwest 0.12.12", + "reqwest 0.12.9", "tempfile", "tokio", ] @@ -8889,7 +8875,7 @@ dependencies = [ "axum", "candid", "ciborium", - "clap 4.5.27", + "clap 4.5.23", "futures", "hex", "ic-agent", @@ -8909,7 +8895,7 @@ dependencies = [ "ic-rosetta-test-utils", "ic-sys", "ic-test-utilities-load-wasm", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "icrc-ledger-agent", "icrc-ledger-types", "indicatif", @@ -8920,7 +8906,7 @@ dependencies = [ "pocket-ic", "proptest 1.6.0", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "rolling-file", "rosetta-core", "rusqlite", @@ -8946,7 +8932,7 @@ version = "0.1.0" dependencies = [ "anyhow", "candid", - "clap 4.5.27", + "clap 4.5.23", "hex", "ic-agent", "ic-crypto-ed25519", @@ -8957,7 +8943,7 @@ dependencies = [ "icrc-ledger-types", "num-bigint 0.4.6", "pocket-ic", - "reqwest 0.12.12", + "reqwest 0.12.9", "rosetta-core", "serde", "tokio", @@ -8972,7 +8958,7 @@ dependencies = [ "candid", "icrc-ledger-types", "pocket-ic", - "reqwest 0.12.12", + "reqwest 0.12.9", "tempfile", "tokio", ] @@ -9010,7 +8996,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_bytes", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -9250,7 +9236,7 @@ dependencies = [ "serde", "strum", "strum_macros", - "thiserror 2.0.11", + "thiserror 2.0.8", "tower 0.5.2", ] @@ -9259,7 +9245,7 @@ name = "ic-interfaces-adapter-client" version = "0.9.0" dependencies = [ "strum_macros", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -9314,7 +9300,7 @@ dependencies = [ "ic-crypto-tree-hash", "ic-types", "phantom_newtype", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -9349,7 +9335,7 @@ dependencies = [ "icp-ledger", "on_wire", "proptest 1.6.0", - "reqwest 0.12.12", + "reqwest 0.12.9", "rusqlite", "serde", "tokio", @@ -9575,7 +9561,7 @@ dependencies = [ "ic-types", "ic_consensus_system_test_utils", "icp-ledger", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "slog", "url", @@ -9589,11 +9575,11 @@ dependencies = [ "assert_matches", "candid", "candid_parser", - "clap 4.5.27", + "clap 4.5.23", "futures", "hex", "maplit", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_json", "sha2 0.10.8", @@ -9756,7 +9742,7 @@ name = "ic-metrics-tool" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", ] [[package]] @@ -9780,7 +9766,7 @@ dependencies = [ "registry-canister", "serde", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", ] @@ -10541,7 +10527,7 @@ version = "0.9.0" dependencies = [ "candid", "canister-test", - "clap 4.5.27", + "clap 4.5.23", "ic-base-types", "ic-canister-client", "ic-interfaces-registry", @@ -10565,7 +10551,7 @@ dependencies = [ name = "ic-nns-inspector" version = "0.1.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "csv", "hex", "ic-base-types", @@ -10801,7 +10787,7 @@ dependencies = [ "quinn", "quinn-udp", "rcgen", - "rustls 0.23.21", + "rustls 0.23.20", "serde", "slog", "tempfile", @@ -10839,7 +10825,7 @@ dependencies = [ "pprof", "prost 0.12.6", "regex", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", ] @@ -10850,7 +10836,7 @@ dependencies = [ "anyhow", "assert_matches", "base64 0.13.1", - "clap 4.5.27", + "clap 4.5.23", "fs_extra", "ic-config", "ic-crypto-node-key-generation", @@ -10881,12 +10867,12 @@ dependencies = [ "pretty_assertions", "prost 0.13.4", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_json", "slog", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "url", "x509-cert", ] @@ -10966,11 +10952,11 @@ dependencies = [ "prost 0.13.4", "quinn", "rstest", - "rustls 0.23.21", + "rustls 0.23.20", "slog", "socket2 0.5.8", "static_assertions", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tokio-metrics", "tokio-util", @@ -10998,7 +10984,7 @@ name = "ic-recovery" version = "0.9.0" dependencies = [ "base64 0.13.1", - "clap 4.5.27", + "clap 4.5.23", "futures", "hex", "ic-artifact-pool", @@ -11028,7 +11014,7 @@ dependencies = [ "ic-test-utilities-types", "ic-types", "prost 0.13.4", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_cbor", "serde_json", @@ -11048,7 +11034,7 @@ version = "0.9.0" dependencies = [ "anyhow", "base64 0.13.1", - "clap 4.5.27", + "clap 4.5.23", "ic-base-types", "ic-crypto-sha2", "ic-crypto-utils-threshold-sig-der", @@ -11066,7 +11052,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "url", ] @@ -11078,7 +11064,7 @@ dependencies = [ "candid", "ic-base-types", "serde", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -11153,7 +11139,7 @@ dependencies = [ "ic-registry-subnet-features", "ic-types", "serde_cbor", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -11202,7 +11188,7 @@ dependencies = [ "ic-registry-transport", "ic-types", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "url", ] @@ -11275,7 +11261,7 @@ dependencies = [ "ic-registry-transport", "ic-sys", "ic-types", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -11290,7 +11276,7 @@ dependencies = [ name = "ic-registry-replicator" version = "0.9.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "ic-config", "ic-crypto-utils-threshold-sig-der", "ic-http-endpoints-metrics", @@ -11375,7 +11361,7 @@ name = "ic-replay" version = "0.9.0" dependencies = [ "candid", - "clap 4.5.27", + "clap 4.5.23", "hex", "ic-artifact-pool", "ic-canister-client", @@ -11429,7 +11415,7 @@ version = "0.9.0" dependencies = [ "assert_cmd", "canister-test", - "clap 4.5.27", + "clap 4.5.23", "criterion", "hex", "ic-artifact-pool", @@ -11685,7 +11671,7 @@ dependencies = [ "async-trait", "base64 0.13.1", "candid", - "clap 4.5.27", + "clap 4.5.23", "dfn_candid", "dfn_protobuf", "futures", @@ -11731,7 +11717,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "rolling-file", "rosetta-core", "rusqlite", @@ -11764,7 +11750,7 @@ dependencies = [ "icp-ledger", "nix 0.24.3", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "rosetta-core", "serde", "serde_bytes", @@ -11845,7 +11831,7 @@ dependencies = [ "serde", "serde_json", "textplots", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", ] @@ -11857,7 +11843,7 @@ dependencies = [ "base64 0.13.1", "candid", "candid-utils", - "clap 4.5.27", + "clap 4.5.23", "cycles-minting-canister", "futures", "hex", @@ -11887,7 +11873,7 @@ dependencies = [ "serde_json", "serde_yaml", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", ] @@ -11903,7 +11889,7 @@ dependencies = [ "canbench-rs", "candid", "candid_parser", - "clap 4.5.27", + "clap 4.5.23", "comparable", "futures", "hex", @@ -11978,7 +11964,7 @@ version = "0.9.0" dependencies = [ "bytes", "candid", - "clap 4.5.27", + "clap 4.5.23", "comparable", "ic-base-types", "ic-nervous-system-proto", @@ -12393,7 +12379,7 @@ name = "ic-starter" version = "0.9.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "ic-config", "ic-logger", "ic-management-canister-types", @@ -12445,7 +12431,7 @@ version = "0.9.0" dependencies = [ "candid", "ciborium", - "clap 4.5.27", + "clap 4.5.23", "hex", "ic-artifact-pool", "ic-base-types", @@ -12605,7 +12591,7 @@ dependencies = [ "prost 0.13.4", "rand 0.8.5", "slog", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tokio-metrics", "tokio-util", @@ -12617,7 +12603,7 @@ dependencies = [ name = "ic-state-tool" version = "0.9.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "hex", "ic-config", "ic-logger", @@ -12642,7 +12628,7 @@ dependencies = [ name = "ic-subnet-splitting" version = "0.9.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "hex", "ic-agent", "ic-base-types", @@ -12681,7 +12667,7 @@ dependencies = [ "prost 0.13.4", "rand 0.8.5", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "wsl", ] @@ -12738,7 +12724,7 @@ dependencies = [ "candid", "canister-test", "chrono", - "clap 4.5.27", + "clap 4.5.23", "config_types", "crossbeam-channel", "cycles-minting-canister", @@ -12816,7 +12802,7 @@ dependencies = [ "ic-types", "ic-types-test-utils", "ic-universal-canister", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "ic-wasm-types", "icp-ledger", "icrc-ledger-types", @@ -12844,7 +12830,7 @@ dependencies = [ "rcgen", "regex", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "ring 0.17.8", "rosetta-core", "rsa", @@ -12862,7 +12848,7 @@ dependencies = [ "strum", "strum_macros", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "time", "tokio", "tokio-util", @@ -13251,7 +13237,7 @@ dependencies = [ "ic_consensus_system_test_utils", "ic_consensus_threshold_sig_system_test_utils", "icrc-ledger-types", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_json", "slog", ] @@ -13291,9 +13277,9 @@ dependencies = [ [[package]] name = "ic-transport-types" -version = "0.39.3" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "979ee7bee5a67150a4c090fb012c93c294a528b4a867bad9a15cc6d01cb4227f" +checksum = "21e2418868dd5857d2a5bac3f1cb6de1aecf2316d380997ef842aec3d8a79d4e" dependencies = [ "candid", "hex", @@ -13304,7 +13290,7 @@ dependencies = [ "serde_cbor", "serde_repr", "sha2 0.10.8", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -13351,7 +13337,7 @@ dependencies = [ "serde_with 1.14.0", "strum", "strum_macros", - "thiserror 2.0.11", + "thiserror 2.0.8", "thousands", ] @@ -13392,9 +13378,9 @@ dependencies = [ [[package]] name = "ic-utils" -version = "0.39.3" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd4274ab690a646a4fb4105428617e9c622255903aad34183bdc892ad8a7cc48" +checksum = "bb1da4a68c45146018b8496c157ad94126b9c202ab4400c6c0a9030c1ef0f0ba" dependencies = [ "async-trait", "candid", @@ -13407,7 +13393,7 @@ dependencies = [ "sha2 0.10.8", "strum", "strum_macros", - "thiserror 2.0.11", + "thiserror 1.0.69", "time", "tokio", ] @@ -13471,7 +13457,7 @@ dependencies = [ "ic-types", "mockall", "rand 0.8.5", - "thiserror 2.0.11", + "thiserror 2.0.8", ] [[package]] @@ -13603,7 +13589,7 @@ checksum = "19fabaeecfe37f24b433c62489242fc54503d98d4cc8d0f9ef7544dfdfc0ddcb" dependencies = [ "anyhow", "candid", - "clap 4.5.27", + "clap 4.5.23", "libflate", "rustc-demangle", "serde", @@ -13643,7 +13629,7 @@ dependencies = [ "byte-unit", "candid", "chrono", - "clap 4.5.27", + "clap 4.5.23", "console 0.11.3", "futures", "hex", @@ -13735,10 +13721,10 @@ dependencies = [ "prometheus", "proptest 1.6.0", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "slog", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "url", ] @@ -13893,7 +13879,7 @@ dependencies = [ "prost 0.13.4", "rand 0.8.5", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "rsa", "serde_json", "slog", @@ -13932,7 +13918,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_cbor", "serde_json", "slog", @@ -13990,7 +13976,7 @@ dependencies = [ "ic_consensus_system_test_utils", "k256", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_bytes", "serde_cbor", "slog", @@ -14015,7 +14001,7 @@ dependencies = [ name = "icp-config" version = "0.9.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "eyre", "ic-config", "ic-replicated-state", @@ -14305,7 +14291,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -14367,9 +14353,9 @@ dependencies = [ [[package]] name = "impl-more" -version = "0.1.9" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" +checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" [[package]] name = "impl-rlp" @@ -14397,7 +14383,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -14419,9 +14405,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -14454,7 +14440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash 0.8.11", - "indexmap 2.7.1", + "indexmap 2.7.0", "is-terminal", "itoa", "log", @@ -14467,22 +14453,22 @@ dependencies = [ [[package]] name = "inferno" -version = "0.12.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692eda1cc790750b9f5a5e3921ef9c117fd5498b97cfacbc910693e5b29002dc" +checksum = "75a5d75fee4d36809e6b021e4b96b686e763d365ffdb03af2bd00786353f84fe" dependencies = [ "ahash 0.8.11", - "clap 4.5.27", + "clap 4.5.23", "crossbeam-channel", "crossbeam-utils", "dashmap 6.1.0", "env_logger", - "indexmap 2.7.1", + "indexmap 2.7.0", "itoa", "log", "num-format", "once_cell", - "quick-xml 0.37.2", + "quick-xml 0.37.1", "rgb", "str_stack", ] @@ -14492,7 +14478,7 @@ name = "inject-files" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "partition_tools", "tempfile", "tokio", @@ -14509,13 +14495,13 @@ dependencies = [ [[package]] name = "insta" -version = "1.42.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513e4067e16e69ed1db5ab56048ed65db32d10ba5fc1217f5393f8f17d8b5a5" +checksum = "7e9ffc4d4892617c50a928c52b2961cb5174b6fc6ebf252b2fac9d21955c48b8" dependencies = [ "console 0.15.10", + "lazy_static", "linked-hash-map", - "once_cell", "similar", ] @@ -14573,9 +14559,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" dependencies = [ "serde", ] @@ -14591,13 +14577,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.14" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f187290c0ed3dfe3f7c85bedddd320949b68fc86ca0ceb71adfb05b3dc3af2a" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ "hermit-abi 0.4.0", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -14689,9 +14675,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -14828,7 +14814,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem 3.0.4", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-pemfile 2.2.0", "secrecy", "serde", @@ -14944,12 +14930,12 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" name = "launch-single-vm" version = "0.1.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "ic-prep", "ic-registry-subnet-type", "ic-system-test-driver", "ic-types", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "slog", "slog-async", @@ -15183,7 +15169,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "libc", "redox_syscall 0.5.8", ] @@ -15241,9 +15227,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.21" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "libc", @@ -15275,9 +15261,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.15" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "linux_kernel_command_line" @@ -15369,9 +15355,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "logos" @@ -15402,7 +15388,7 @@ dependencies = [ "proc-macro2", "quote", "regex-syntax 0.6.29", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -15428,19 +15414,6 @@ dependencies = [ "logos-codegen", ] -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if 1.0.0", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - [[package]] name = "lru" version = "0.7.8" @@ -15665,7 +15638,7 @@ dependencies = [ "ic-registry-subnet-type", "ic-system-test-driver", "ic-types", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "itertools 0.12.1", "rand 0.8.5", "rand_chacha 0.3.1", @@ -15735,9 +15708,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -15806,7 +15779,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -15835,23 +15808,25 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.10" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f" dependencies = [ "async-lock", + "async-trait", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", - "event-listener 5.4.0", + "event-listener 5.3.1", "futures-util", - "loom", + "once_cell", "parking_lot 0.12.3", - "portable-atomic", + "quanta", "rustc_version", "smallvec", "tagptr", "thiserror 1.0.69", + "triomphe", "uuid", ] @@ -15904,9 +15879,9 @@ checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" [[package]] name = "neli" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93062a0dce6da2517ea35f301dfc88184ce18d3601ec786a727a87bf535deca9" +checksum = "1100229e06604150b3becd61a4965d5c70f3be1759544ea7274166f4be41ef43" dependencies = [ "byteorder", "libc", @@ -15916,9 +15891,9 @@ dependencies = [ [[package]] name = "neli-proc-macros" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8034b7fbb6f9455b2a96c19e6edf8dc9fc34c70449938d8ee3b4df363f61fe" +checksum = "c168194d373b1e134786274020dae7fc5513d565ea2ebb9bc9ff17ffb69106d4" dependencies = [ "either", "proc-macro2", @@ -15988,12 +15963,12 @@ dependencies = [ "ic-test-utilities", "ic-test-utilities-types", "ic-types", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "proxy_canister", "rand 0.8.5", "rand_chacha 0.3.1", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "slog", "tokio", "url", @@ -16010,7 +15985,7 @@ name = "nft_exporter" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "serde", "serde_json", ] @@ -16058,7 +16033,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "cfg-if 1.0.0", "libc", "memoffset 0.9.1", @@ -16070,7 +16045,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "cfg-if 1.0.0", "cfg_aliases", "libc", @@ -16120,7 +16095,7 @@ dependencies = [ "on_wire", "prost 0.13.4", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_cbor", "slog", "tokio", @@ -16349,7 +16324,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -16369,13 +16344,13 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.7" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "crc32fast", "hashbrown 0.15.2", - "indexmap 2.7.1", + "indexmap 2.7.0", "memchr", ] @@ -16641,7 +16616,7 @@ dependencies = [ "async-trait", "backoff", "candid", - "clap 4.5.27", + "clap 4.5.23", "env-file-reader", "exec", "get_if_addrs", @@ -16751,7 +16726,7 @@ dependencies = [ "ic-types", "ic_consensus_system_test_utils", "itertools 0.12.1", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "slog", "tokio", @@ -16977,7 +16952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.11", + "thiserror 2.0.8", "ucd-trie", ] @@ -17001,7 +16976,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -17032,7 +17007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.7.1", + "indexmap 2.7.0", ] [[package]] @@ -17057,12 +17032,12 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", - "phf_shared 0.11.3", + "phf_shared 0.11.2", ] [[package]] @@ -17087,25 +17062,25 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ - "phf_shared 0.11.3", + "phf_shared 0.11.2", "rand 0.8.5", ] [[package]] name = "phf_macros" -version = "0.11.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", + "phf_generator 0.11.2", + "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -17114,16 +17089,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "siphasher 0.3.11", + "siphasher", ] [[package]] name = "phf_shared" -version = "0.11.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ - "siphasher 1.0.1", + "siphasher", ] [[package]] @@ -17134,29 +17109,29 @@ checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" [[package]] name = "pin-project" -version = "1.1.8" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.8" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -17270,7 +17245,7 @@ dependencies = [ "ic-transport-types", "k256", "lazy_static", - "reqwest 0.12.12", + "reqwest 0.12.9", "schemars", "serde", "serde_bytes", @@ -17280,7 +17255,7 @@ dependencies = [ "slog", "strum", "strum_macros", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tracing", "tracing-appender", @@ -17305,7 +17280,7 @@ dependencies = [ "bitcoincore-rpc", "bytes", "candid", - "clap 4.5.27", + "clap 4.5.23", "ctrlc", "flate2", "form_urlencoded", @@ -17354,7 +17329,7 @@ dependencies = [ "ic-test-utilities", "ic-test-utilities-registry", "ic-types", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "ic-utils-thread", "ic-validator-ingress-message", "itertools 0.12.1", @@ -17363,7 +17338,7 @@ dependencies = [ "rand 0.8.5", "rcgen", "registry-canister", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_cbor", "serde_json", @@ -17559,12 +17534,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.29" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -17642,9 +17617,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -17669,7 +17644,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "hex", "lazy_static", "procfs-core", @@ -17682,7 +17657,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "hex", ] @@ -17743,7 +17718,7 @@ checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set 0.8.0", "bit-vec 0.8.0", - "bitflags 2.8.0", + "bitflags 2.6.0", "lazy_static", "num-traits", "rand 0.8.5", @@ -17763,7 +17738,7 @@ checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -17803,7 +17778,7 @@ dependencies = [ "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.96", + "syn 2.0.90", "tempfile", ] @@ -17823,7 +17798,7 @@ dependencies = [ "prost 0.13.4", "prost-types 0.13.4", "regex", - "syn 2.0.96", + "syn 2.0.90", "tempfile", ] @@ -17837,7 +17812,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -17850,7 +17825,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -17935,9 +17910,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8324e531de91a3c25021a30fb7862d39cc516b61fbb801176acb5ff279ea887b" +checksum = "403a1a95f4c18a45c86c7bff13df00347afd0abcbf2e54af273c837339ffcf77" dependencies = [ "cranelift-bitset", "log", @@ -17946,9 +17921,9 @@ dependencies = [ [[package]] name = "quanta" -version = "0.12.5" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" +checksum = "773ce68d0bb9bc7ef20be3536ffe94e223e1f365bd374108b2659fac0c65cfe6" dependencies = [ "crossbeam-utils", "libc", @@ -17976,9 +17951,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.2" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" dependencies = [ "memchr", ] @@ -17994,9 +17969,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.0", - "rustls 0.23.21", + "rustls 0.23.20", "socket2 0.5.8", - "thiserror 2.0.11", + "thiserror 2.0.8", "tokio", "tracing", ] @@ -18012,10 +17987,10 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash 2.1.0", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.11", + "thiserror 2.0.8", "tinyvec", "tracing", "web-time", @@ -18037,9 +18012,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.38" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -18262,7 +18237,7 @@ version = "0.9.0" dependencies = [ "anyhow", "candid", - "clap 4.5.27", + "clap 4.5.23", "ic-agent", "k256", "rate-limits-api", @@ -18316,7 +18291,7 @@ dependencies = [ "serde_cbor", "serde_json", "strum", - "thiserror 2.0.11", + "thiserror 2.0.8", "uuid", ] @@ -18333,11 +18308,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.3.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6928fa44c097620b706542d428957635951bade7143269085389d42c8a4927e" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", ] [[package]] @@ -18362,9 +18337,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" dependencies = [ "pem 3.0.4", "ring 0.17.8", @@ -18398,7 +18373,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", ] [[package]] @@ -18662,9 +18637,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.12" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", @@ -18688,7 +18663,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -18700,7 +18675,6 @@ dependencies = [ "tokio-rustls 0.26.1", "tokio-socks", "tokio-util", - "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -18958,7 +18932,7 @@ dependencies = [ "on_wire", "prost 0.13.4", "rand 0.8.5", - "reqwest 0.12.12", + "reqwest 0.12.9", "rosetta-core", "serde", "serde_json", @@ -19013,7 +18987,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.96", + "syn 2.0.90", "unicode-ident", ] @@ -19136,11 +19110,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.44" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "errno 0.3.10", "libc", "linux-raw-sys", @@ -19175,9 +19149,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.21" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "brotli 7.0.0", "brotli-decompressor", @@ -19211,7 +19185,7 @@ dependencies = [ "ring 0.17.8", "serde", "serde_json", - "thiserror 2.0.11", + "thiserror 2.0.8", "webpki-roots 0.26.7", "x509-parser", ] @@ -19238,7 +19212,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.2.0", + "security-framework 3.1.0", ] [[package]] @@ -19279,7 +19253,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.21", + "rustls 0.23.20", "rustls-native-certs 0.7.3", "rustls-platform-verifier-android", "rustls-webpki 0.102.8", @@ -19318,9 +19292,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.19" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "rusty-fork" @@ -19382,7 +19356,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -19401,7 +19375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" dependencies = [ "dyn-clone", - "indexmap 2.7.1", + "indexmap 2.7.0", "schemars_derive", "serde", "serde_json", @@ -19416,7 +19390,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -19475,7 +19449,7 @@ dependencies = [ "ic-system-test-driver", "ic_consensus_system_test_utils", "nns_dapp", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde_json", "slog", ] @@ -19577,7 +19551,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -19587,11 +19561,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.2.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -19600,9 +19574,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.14.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -19614,7 +19588,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "cssparser", "derive_more 0.99.18", "fxhash", @@ -19629,18 +19603,18 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.25" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.217" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] @@ -19698,13 +19672,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -19715,14 +19689,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.137" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -19771,7 +19745,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -19794,7 +19768,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -19856,7 +19830,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -19865,7 +19839,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.1", + "indexmap 2.7.0", "itoa", "ryu", "serde", @@ -19886,7 +19860,7 @@ name = "setupos-disable-checks" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "indoc", "linux_kernel_command_line", "partition_tools", @@ -19900,7 +19874,7 @@ name = "setupos-inject-configuration" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "config", "partition_tools", "serde", @@ -19916,7 +19890,7 @@ name = "setupos_tool" version = "1.0.0" dependencies = [ "anyhow", - "clap 4.5.27", + "clap 4.5.23", "config", "config_types", "deterministic_ips", @@ -20032,19 +20006,19 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "similar" -version = "2.7.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" [[package]] name = "simple_asn1" -version = "0.6.3" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint 0.4.6", "num-traits", - "thiserror 2.0.11", + "thiserror 1.0.69", "time", ] @@ -20074,12 +20048,6 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" -[[package]] -name = "siphasher" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" - [[package]] name = "slab" version = "0.4.9" @@ -20209,7 +20177,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20237,7 +20205,7 @@ dependencies = [ "ic-system-test-driver", "ic-types", "ic-universal-canister", - "ic-utils 0.39.3", + "ic-utils 0.39.0", "ic_consensus_system_test_utils", "icp-ledger", "icrc-ledger-agent", @@ -20295,7 +20263,7 @@ dependencies = [ "ic-registry-subnet-type", "ic-system-test-driver", "ic-types", - "reqwest 0.12.12", + "reqwest 0.12.9", "slog", ] @@ -20475,7 +20443,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.2.0", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20487,7 +20455,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.3.0", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20498,7 +20466,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20509,7 +20477,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20531,7 +20499,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20559,9 +20527,9 @@ checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" [[package]] name = "symbolic-common" -version = "12.13.3" +version = "12.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a4dfe4bbeef59c1f32fc7524ae7c95b9e1de5e79a43ce1604e181081d71b0c" +checksum = "cd33e73f154e36ec223c18013f7064a2c120f1162fc086ac9933542def186b00" dependencies = [ "debugid", "memmap2", @@ -20571,9 +20539,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.13.3" +version = "12.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cf6a95abff97de4d7ff3473f33cacd38f1ddccad5c1feab435d6760300e3b6" +checksum = "89e51191290147f071777e37fe111800bb82a9059f9c95b19d2dd41bfeddf477" dependencies = [ "rustc-demangle", "symbolic-common", @@ -20592,9 +20560,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.96" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -20646,7 +20614,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20677,11 +20645,11 @@ dependencies = [ "anyhow", "async-trait", "axum", - "clap 4.5.27", + "clap 4.5.23", "http 1.2.0", "itertools 0.12.1", - "reqwest 0.12.12", - "thiserror 2.0.11", + "reqwest 0.12.9", + "thiserror 2.0.8", "tokio", "url", ] @@ -20786,13 +20754,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.15.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if 1.0.0", "fastrand", - "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -20863,7 +20830,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta 0.2.0", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20875,7 +20842,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta 0.3.0", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -20915,7 +20882,7 @@ dependencies = [ "ic_consensus_system_test_utils", "nns_dapp", "os_qualification_utils", - "reqwest 0.12.12", + "reqwest 0.12.9", "serde", "serde_json", "slog", @@ -20950,11 +20917,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "08f5383f3e0071702bf93ab5ee99b52d26936be9dedd9413067cbdcddcb6141a" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.8", ] [[package]] @@ -20965,18 +20932,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "f2f357fcec90b3caef6623a099691be676d033b40a058ac95d2a6ade6fa0c943" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -21099,9 +21066,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -21152,14 +21119,14 @@ checksum = "8d9ef545650e79f30233c0003bcc2504d7efac6dad25fca40744de773fe2049c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] name = "tokio" -version = "1.43.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -21185,13 +21152,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -21233,7 +21200,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.21", + "rustls 0.23.20", "tokio", ] @@ -21330,7 +21297,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.7.1", + "indexmap 2.7.0", "toml_datetime", "winnow", ] @@ -21376,7 +21343,7 @@ dependencies = [ "prost-build 0.13.4", "prost-types 0.13.4", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -21408,7 +21375,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.7.1", + "indexmap 2.7.0", "pin-project-lite", "slab", "sync_wrapper 1.0.2", @@ -21426,7 +21393,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "base64 0.21.7", - "bitflags 2.8.0", + "bitflags 2.6.0", "bytes", "http 1.2.0", "http-body 1.0.1", @@ -21445,7 +21412,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "async-compression", - "bitflags 2.8.0", + "bitflags 2.6.0", "bytes", "futures-core", "http 1.2.0", @@ -21547,7 +21514,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -21687,6 +21654,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "triomphe" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" + [[package]] name = "trust-dns-proto" version = "0.22.0" @@ -21809,10 +21782,11 @@ dependencies = [ [[package]] name = "ulid" -version = "1.1.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f294bff79170ed1c5633812aff1e565c35d993a36e757f9bc0accf5eec4e6045" +checksum = "04f903f293d11f31c0c29e4148f6dc0d033a7f80cebc0282bea147611667d289" dependencies = [ + "getrandom", "rand 0.8.5", "web-time", ] @@ -21825,9 +21799,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicase" -version = "2.8.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" @@ -21979,9 +21953,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.12.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", "serde", @@ -21989,9 +21963,9 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.1" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "vcpkg" @@ -22038,7 +22012,7 @@ dependencies = [ name = "vsock_guest" version = "1.0.0" dependencies = [ - "clap 4.5.27", + "clap 4.5.23", "vsock_lib", ] @@ -22055,7 +22029,7 @@ version = "1.0.0" dependencies = [ "anyhow", "regex", - "reqwest 0.12.12", + "reqwest 0.12.9", "rusb", "serde", "serde_json", @@ -22167,35 +22141,34 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if 1.0.0", "once_cell", - "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.100" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -22206,9 +22179,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -22216,25 +22189,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-encoder" @@ -22267,12 +22237,12 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.224.0" +version = "0.222.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7249cf8cb0c6b9cb42bce90c0a5feb276fbf963fa385ff3d818ab3d90818ed6" +checksum = "3432682105d7e994565ef928ccf5856cf6af4ba3dddebedb737f61caed70f956" dependencies = [ "leb128", - "wasmparser 0.224.0", + "wasmparser 0.222.0", ] [[package]] @@ -22295,9 +22265,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d28bc49ba1e5c5b61ffa7a2eace10820443c4b7d1c0b144109261d14570fdf8" dependencies = [ "ahash 0.8.11", - "bitflags 2.8.0", + "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap 2.7.1", + "indexmap 2.7.0", "semver", "serde", ] @@ -22309,9 +22279,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca917a21307d3adf2b9857b94dd05ebf8496bdcff4437a9b9fb3899d3e6c74e7" dependencies = [ "ahash 0.8.11", - "bitflags 2.8.0", + "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap 2.7.1", + "indexmap 2.7.0", "semver", "serde", ] @@ -22322,21 +22292,21 @@ version = "0.221.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9845c470a2e10b61dd42c385839cdd6496363ed63b5c9e420b5488b77bd22083" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.6.0", "hashbrown 0.15.2", - "indexmap 2.7.1", + "indexmap 2.7.0", "semver", "serde", ] [[package]] name = "wasmparser" -version = "0.224.0" +version = "0.222.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65881a664fdd43646b647bb27bf186ab09c05bf56779d40aed4c6dce47d423f5" +checksum = "4adf50fde1b1a49c1add6a80d47aea500c88db70551805853aa8b88f3ea27ab5" dependencies = [ - "bitflags 2.8.0", - "indexmap 2.7.1", + "bitflags 2.6.0", + "indexmap 2.7.0", "semver", ] @@ -22364,17 +22334,17 @@ dependencies = [ [[package]] name = "wasmtime" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd30973c65eceb0f37dfcc430d83abd5eb24015fdfcab6912f52949287e04f0" +checksum = "f639ecae347b9a2227e453a7b7671e84370a0b61f47a15e0390fe9b7725e47b3" dependencies = [ "anyhow", - "bitflags 2.8.0", + "bitflags 2.6.0", "bumpalo", "cc", "cfg-if 1.0.0", "hashbrown 0.14.5", - "indexmap 2.7.1", + "indexmap 2.7.0", "libc", "libm", "log", @@ -22408,23 +22378,23 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c21dd30d1f3f93ee390ac1a7ec304ecdbfdab6390e1add41a1f52727b0992b" +checksum = "882a18800471cfc063c8b3ccf75723784acc3fd534009ac09421f2fac2fcdcec" dependencies = [ "cfg-if 1.0.0", ] [[package]] name = "wasmtime-component-macro" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f948a6ef3119d52c9f12936970de28ddf3f9bea04bc65571f4a92d2e5ab38f4" +checksum = "eb5c0a77c9e1927c3d471f53cc13767c3d3438e5d5ffd394e3eb31c86445fd60" dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -22432,15 +22402,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9275aa01ceaaa2fa6c0ecaa5267518d80b9d6e9ae7c7ea42f4c6e073e6a69ef" +checksum = "43702ca98bf5162eca0573db691ed9ecd36d716f8c6688410fe26ec16b6f9bcb" [[package]] name = "wasmtime-cranelift" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0701a44a323267aae4499672dae422b266cee3135a23b640972ec8c0e10a44a2" +checksum = "20070aa5b75080a8932ec328419faf841df2bc6ceb16b55b0df2b952098392a2" dependencies = [ "anyhow", "cfg-if 1.0.0", @@ -22463,15 +22433,15 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "264c968c1b81d340355ece2be0bc31a10f567ccb6ce08512c3b7d10e26f3cbe5" +checksum = "2604ddb24879d4dc1dedcb7081d7a8e017259bce916fdae097a97db52cbaab80" dependencies = [ "anyhow", "cranelift-bitset", "cranelift-entity", "gimli 0.31.1", - "indexmap 2.7.1", + "indexmap 2.7.0", "log", "object", "postcard", @@ -22486,9 +22456,9 @@ dependencies = [ [[package]] name = "wasmtime-fiber" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78505221fd5bd7b07b4e1fa2804edea49dc231e626ad6861adc8f531812973e6" +checksum = "98593412d2b167ebe2b59d4a17a184978a72f976b53b3a0ec05629451079ac1d" dependencies = [ "anyhow", "cc", @@ -22501,9 +22471,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bedb677ca1b549d98f95e9e1f9251b460090d99a2c196a0614228c064bf2e59" +checksum = "d40d7722b9e1fbeae135715710a8a2570b1e6cf72b74dd653962d89831c6c70d" dependencies = [ "anyhow", "cfg-if 1.0.0", @@ -22513,26 +22483,26 @@ dependencies = [ [[package]] name = "wasmtime-slab" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "564905638c132c275d365c1fa074f0b499790568f43148d29de84ccecfb5cb31" +checksum = "8579c335220b4ece9aa490a0e8b46de78cd342b195ab21ff981d095e14b52383" [[package]] name = "wasmtime-versioned-export-macros" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91092e6cf77390eeccee273846a9327f3e8f91c3c6280f60f37809f0e62d29" +checksum = "d7de0a56fb0a69b185968f2d7a9ba54750920a806470dff7ad8de91ac06d277e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] name = "wasmtime-winch" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b111d909dc604c741bd8ac2f4af373eaa5c68c34b5717271bcb687688212cef8" +checksum = "abd309943c443f5590d12f9aba9ba63c481091c955a0a14de0c2a9e0e3aaeca9" dependencies = [ "anyhow", "cranelift-codegen", @@ -22547,13 +22517,13 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f38f7a5eb2f06f53fe943e7fb8bf4197f7cf279f1bc52c0ce56e9d3ffd750a4" +checksum = "969f83022dac3435d6469edb582ceed04cfe32aa44dc3ef16e5cb55574633df8" dependencies = [ "anyhow", "heck 0.5.0", - "indexmap 2.7.1", + "indexmap 2.7.0", "wit-parser", ] @@ -22572,31 +22542,31 @@ dependencies = [ [[package]] name = "wast" -version = "224.0.0" +version = "222.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d722a51e62b669d17e5a9f6bc8ec210178b37d869114355aa46989686c5c6391" +checksum = "5ce7191f4b7da0dd300cc32476abae6457154e4625d9b1bc26890828a9a26f6e" dependencies = [ "bumpalo", "leb128", "memchr", "unicode-width 0.2.0", - "wasm-encoder 0.224.0", + "wasm-encoder 0.222.0", ] [[package]] name = "wat" -version = "1.224.0" +version = "1.222.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71dece6a7dd5bcbcf8d256606c7fb3faa36286d46bf3f98185407719a5ceede2" +checksum = "8fde61b4b52f9a84ae31b5e8902a2cd3162ea45d8bf564c729c3288fe52f4334" dependencies = [ - "wast 224.0.0", + "wast 222.0.0", ] [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -22717,9 +22687,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "28.0.1" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6232f40a795be2ce10fc761ed3b403825126a60d12491ac556ea104a932fd18a" +checksum = "9110decc2983ed94de904804dcd979ba59cbabc78a94fec6b1d8468ec513d0f6" dependencies = [ "anyhow", "cranelift-codegen", @@ -22732,16 +22702,6 @@ dependencies = [ "wasmtime-environ", ] -[[package]] -name = "windows" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" -dependencies = [ - "windows-core 0.58.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-core" version = "0.52.0" @@ -22751,41 +22711,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-strings", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-implement" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - -[[package]] -name = "windows-interface" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - [[package]] name = "windows-registry" version = "0.2.0" @@ -22966,9 +22891,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.24" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -22997,7 +22922,7 @@ checksum = "fbe1538eea6ea5ddbe5defd0dc82539ad7ba751e1631e9185d24a931f0a5adc8" dependencies = [ "anyhow", "id-arena", - "indexmap 2.7.1", + "indexmap 2.7.0", "log", "semver", "serde", @@ -23084,9 +23009,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.4.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", "linux-raw-sys", @@ -23183,7 +23108,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", "synstructure", ] @@ -23205,7 +23130,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -23225,7 +23150,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", "synstructure", ] @@ -23246,7 +23171,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] @@ -23268,7 +23193,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.90", ] [[package]] diff --git a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh index f105b3bfa64..d1c173f2f31 100755 --- a/ic-os/components/early-boot/setup-hostname/setup-hostname.sh +++ b/ic-os/components/early-boot/setup-hostname/setup-hostname.sh @@ -55,7 +55,7 @@ function read_config_variables() { } function construct_hostname() { - if [ "${config_hostname}" != "" ] && [ "${config_hostname}" != "null" ]; then + if [ -n "${config_hostname}" ]; then HOSTNAME=${config_hostname} write_log "Using manually configured hostname: ${HOSTNAME}" elif [[ -r ${FILE} && $(cat ${FILE}) != "" ]]; then diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 93496b6661d..694bb13aeba 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -88,10 +88,7 @@ function assemble_config_media() { fi cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && "$ipv4_address" != "null" \ - && -n "$ipv4_prefix_length" && "$ipv4_prefix_length" != "null" \ - && -n "$ipv4_gateway" && "$ipv4_gateway" != "null" \ - && -n "$domain_name" && "$domain_name" != "null" ]]; then + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain_name}") diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index fb150311eaf..89ea53fa8b4 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -84,10 +84,7 @@ function assemble_config_media() { fi cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && "$ipv4_address" != "null" \ - && -n "$ipv4_prefix_length" && "$ipv4_prefix_length" != "null" \ - && -n "$ipv4_gateway" && "$ipv4_gateway" != "null" \ - && -n "$domain_name" && "$domain_name" != "null" ]]; then + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") cmd+=(--domain "${domain_name}") diff --git a/ic-os/components/misc/config/config.sh b/ic-os/components/misc/config/config.sh index 5cc0f222b2c..106a71bdeb4 100644 --- a/ic-os/components/misc/config/config.sh +++ b/ic-os/components/misc/config/config.sh @@ -5,9 +5,18 @@ # Retrieves a value from the config.json file using a JSON path. # Arguments: # $1 - JSON path to the desired value (e.g., '.icos_settings.nns_urls') -# Note: If the key is not found, this function will return null. +# Returns: +# If key is not found or value is "null", returns empty string. +# Otherwise, returns value. function get_config_value() { local CONFIG_FILE="/boot/config/config.json" local key=$1 - jq -r "${key}" "${CONFIG_FILE}" + + local value=$(jq -r "${key}" "${CONFIG_FILE}") + + if [[ "${value}" == "null" ]]; then + echo "" + else + echo "${value}" + fi } From 925138a642e162ea805e986803b68fa9021f6042 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Tue, 28 Jan 2025 20:07:50 +0000 Subject: [PATCH 240/241] Separate domain_name from ipv4 parameters --- .../generate-guestos-config/dev-generate-guestos-config.sh | 4 +++- .../generate-guestos-config/generate-guestos-config.sh | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 694bb13aeba..d3186ca219c 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -88,9 +88,11 @@ function assemble_config_media() { fi cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") + fi + if [[ -n "$domain_name" ]]; then cmd+=(--domain "${domain_name}") fi if [[ -n "$node_reward_type" ]]; then diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 89ea53fa8b4..6fadc16bb75 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -84,9 +84,11 @@ function assemble_config_media() { fi cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") - if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain_name" ]]; then + if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" ]]; then cmd+=(--ipv4_address "${ipv4_address}/${ipv4_prefix_length}") cmd+=(--ipv4_gateway "${ipv4_gateway}") + fi + if [[ -n "$domain_name" ]]; then cmd+=(--domain "${domain_name}") fi if [[ -n "$node_reward_type" ]]; then From 65468495378e318481d11f953fc3d2c0738b0ccd Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Tue, 28 Jan 2025 20:14:45 +0000 Subject: [PATCH 241/241] Automatically updated Cargo*.lock --- Cargo.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ea06f55e2b5..c38f1484aa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12725,6 +12725,7 @@ dependencies = [ "canister-test", "chrono", "clap 4.5.23", + "config", "config_types", "crossbeam-channel", "cycles-minting-canister", @@ -14931,6 +14932,8 @@ name = "launch-single-vm" version = "0.1.0" dependencies = [ "clap 4.5.23", + "config", + "config_types", "ic-prep", "ic-registry-subnet-type", "ic-system-test-driver",