From bb7d862c97211e9c2fa051ca8f6ca74a48a14cf6 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 1 Nov 2024 18:43:09 +0100 Subject: [PATCH 001/148] Move config to its own utils --- libsysinspect/src/mdescr/mspec.rs | 39 ++--------------------------- libsysinspect/src/util/cfg.rs | 41 +++++++++++++++++++++++++++++++ libsysinspect/src/util/mod.rs | 1 + 3 files changed, 44 insertions(+), 37 deletions(-) create mode 100644 libsysinspect/src/util/cfg.rs diff --git a/libsysinspect/src/mdescr/mspec.rs b/libsysinspect/src/mdescr/mspec.rs index 7faeea95..4fc0a23e 100644 --- a/libsysinspect/src/mdescr/mspec.rs +++ b/libsysinspect/src/mdescr/mspec.rs @@ -1,9 +1,7 @@ use super::{datapatch, mspecdef::ModelSpec}; -use crate::SysinspectError; -use nix::unistd::Uid; +use crate::{util::cfg::select_config, SysinspectError}; use serde_yaml::Value; use std::{ - env::{self}, fs::{self}, path::{Path, PathBuf}, }; @@ -11,8 +9,6 @@ use walkdir::WalkDir; pub const MODEL_INDEX: &str = "model.cfg"; pub const MODEL_FILE_EXT: &str = ".cfg"; -pub const APP_CONF: &str = "sysinspect.conf"; -pub const APP_DOTCONF: &str = ".sysinspect"; /// Spec loader object struct SpecLoader { @@ -104,37 +100,6 @@ impl SpecLoader { Ok(base) } - /// Select app conf - fn select_config(&self) -> Result { - // Current - let cfp: PathBuf = env::current_dir()?.canonicalize()?.join(APP_CONF); - if cfp.exists() { - return Ok(cfp); - } - - // Dot-file - let cfp = env::var_os("HOME").map(PathBuf::from).or_else(|| { - #[cfg(unix)] - { - Some(PathBuf::from(format!("/home/{}", Uid::current()))) - } - }); - if let Some(cfp) = cfp { - let cfp = cfp.join(APP_DOTCONF); - if cfp.exists() { - return Ok(cfp); - } - } - - // Global conf - let cfp = PathBuf::from(format!("/etc/{}", APP_CONF)); - if cfp.exists() { - return Ok(cfp); - } - - Err(SysinspectError::ConfigError("No config has been found".to_string())) - } - /// Load model spec by merging all the data parts and validating /// its content. fn load(&mut self) -> Result { @@ -157,7 +122,7 @@ impl SpecLoader { } // Load app config and merge to the main model - base.push(serde_yaml::from_str::(&fs::read_to_string(self.select_config()?)?)?); + base.push(serde_yaml::from_str::(&fs::read_to_string(select_config()?)?)?); let mut base = self.merge_parts(&mut base)?; if !iht.is_empty() { diff --git a/libsysinspect/src/util/cfg.rs b/libsysinspect/src/util/cfg.rs new file mode 100644 index 00000000..dd4f301a --- /dev/null +++ b/libsysinspect/src/util/cfg.rs @@ -0,0 +1,41 @@ +/* +Config reader + */ + +use crate::SysinspectError; +use nix::unistd::Uid; +use std::{env, path::PathBuf}; + +pub const APP_CONF: &str = "sysinspect.conf"; +pub const APP_DOTCONF: &str = ".sysinspect"; + +/// Select app conf +pub fn select_config() -> Result { + // Current + let cfp: PathBuf = env::current_dir()?.canonicalize()?.join(APP_CONF); + if cfp.exists() { + return Ok(cfp); + } + + // Dot-file + let cfp = env::var_os("HOME").map(PathBuf::from).or_else(|| { + #[cfg(unix)] + { + Some(PathBuf::from(format!("/home/{}", Uid::current()))) + } + }); + if let Some(cfp) = cfp { + let cfp = cfp.join(APP_DOTCONF); + if cfp.exists() { + return Ok(cfp); + } + } + + // Global conf + let cfp = PathBuf::from(format!("/etc/{}", APP_CONF)); + if cfp.exists() { + return Ok(cfp); + } + + Err(SysinspectError::ConfigError("No config has been found".to_string())) +} diff --git a/libsysinspect/src/util/mod.rs b/libsysinspect/src/util/mod.rs index 77c2b205..3d9b2b17 100644 --- a/libsysinspect/src/util/mod.rs +++ b/libsysinspect/src/util/mod.rs @@ -1 +1,2 @@ +pub mod cfg; pub mod dataconv; From 67a57cd15316b94ed131c338d48a8efd40fd4163 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 19:07:47 +0100 Subject: [PATCH 002/148] Add a very simple Minion (no proto yet) --- sysminion/Cargo.toml | 28 ++++++++++++++ sysminion/src/clidef.rs | 74 +++++++++++++++++++++++++++++++++++ sysminion/src/config.rs | 33 ++++++++++++++++ sysminion/src/main.rs | 55 ++++++++++++++++++++++++++ sysminion/src/minion.rs | 86 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 276 insertions(+) create mode 100644 sysminion/Cargo.toml create mode 100644 sysminion/src/clidef.rs create mode 100644 sysminion/src/config.rs create mode 100644 sysminion/src/main.rs create mode 100644 sysminion/src/minion.rs diff --git a/sysminion/Cargo.toml b/sysminion/Cargo.toml new file mode 100644 index 00000000..e684649e --- /dev/null +++ b/sysminion/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "sysminion" +version = "0.1.0" +edition = "2021" + +[dependencies] +clap = { version = "4.5.20", features = ["unstable-styles"] } +colored = "2.1.0" +ed25519-dalek = { version = "2.1.1", features = [ + "asm", + "batch", + "digest", + "merlin", + "pem", + "pkcs8", + "rand_core", + "signature", + "serde", +] } +rand = "0.8.5" +rustls = "0.23.16" +rustls-pemfile = "2.2.0" +tokio = { version = "1.41.0", features = ["full"] } +libsysinspect = { path = "../libsysinspect" } +log = "0.4.22" +serde_yaml = "0.9.34" +serde = { version = "1.0.214", features = ["derive"] } +serde_json = "1.0.132" diff --git a/sysminion/src/clidef.rs b/sysminion/src/clidef.rs new file mode 100644 index 00000000..0602733f --- /dev/null +++ b/sysminion/src/clidef.rs @@ -0,0 +1,74 @@ +use clap::builder::styling; +use clap::{Arg, ArgAction, Command}; +use colored::Colorize; + +/// Define CLI arguments and styling +pub fn cli(version: &'static str, appname: &'static str) -> Command { + let styles = styling::Styles::styled() + .header(styling::AnsiColor::Yellow.on_default()) + .usage(styling::AnsiColor::Yellow.on_default()) + .literal(styling::AnsiColor::BrightGreen.on_default()) + .placeholder(styling::AnsiColor::BrightMagenta.on_default()); + + Command::new(appname) + .version(version) + .about(format!("{} - {}", appname.bright_magenta().bold(), "is an agent client on a remote device")) + .override_usage(format!("{} [OPTIONS]", appname)) + + // Config + .arg( + Arg::new("config") + .short('c') + .long("config") + .help("Alternative path to the config") + ) + .arg( + Arg::new("master") + .short('r') + .long("master") + .help("Register to the master by Id") // XXX: This must be a key fingerprint in a future + ) + .arg( + Arg::new("start") + .long("start") + .action(ArgAction::SetTrue) + .help("Start minion") + ) + + .next_help_heading("Info") + .arg( + Arg::new("status") + .long("status") + .action(ArgAction::SetTrue) + .help("Show minion status") + ) + + + // Other + .next_help_heading("Other") + .arg( + Arg::new("debug") + .short('d') + .long("debug") + .action(ArgAction::Count) + .help("Set debug mode for more verbose output. Increase this flag for more verbosity."), + ) + .arg( + Arg::new("help") + .short('h') + .long("help") + .action(ArgAction::SetTrue) + .help("Display help"), + ) + .arg( + Arg::new("version") + .short('v') + .long("version") + .action(ArgAction::SetTrue) + .help("Get current version."), + ) + .disable_help_flag(true) // Otherwise it is displayed in a wrong position + .disable_version_flag(true) + .disable_colored_help(false) + .styles(styles) +} diff --git a/sysminion/src/config.rs b/sysminion/src/config.rs new file mode 100644 index 00000000..2bdad964 --- /dev/null +++ b/sysminion/src/config.rs @@ -0,0 +1,33 @@ +use libsysinspect::{intp::functions::get_by_namespace, SysinspectError}; +use serde::{Deserialize, Serialize}; +use serde_yaml::{from_str, from_value, Value}; +use std::{fs, path::PathBuf}; + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +pub struct MinionConfig { + #[serde(rename = "master.ip")] + master_ip: String, + + #[serde(rename = "master.port")] + master_port: u32, +} + +impl MinionConfig { + pub fn new(p: PathBuf) -> Result { + let cp = p.as_os_str().to_str().unwrap_or_default(); + if !p.exists() { + return Err(SysinspectError::ConfigError(format!("File not found: {}", cp))); + } + + if let Some(cfgv) = get_by_namespace(Some(from_str::(&fs::read_to_string(&p)?)?), "config.minion") { + return Ok(from_value::(cfgv)?); + } + + Err(SysinspectError::ConfigError(format!("Unable to read config at: {}", cp))) + } + + /// Return master addr + pub fn master(&self) -> String { + format!("{}:{}", self.master_ip, self.master_port) + } +} diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs new file mode 100644 index 00000000..b8a017bb --- /dev/null +++ b/sysminion/src/main.rs @@ -0,0 +1,55 @@ +mod clidef; +mod config; +mod minion; + +use clidef::cli; +use libsysinspect::logger; +use log::LevelFilter; +use std::{env, path::PathBuf}; + +static APPNAME: &str = "sysminion"; +static VERSION: &str = "0.0.1"; +static LOGGER: logger::STDOUTLogger = logger::STDOUTLogger; + +#[tokio::main] +async fn main() -> std::io::Result<()> { + let mut cli = cli(VERSION, APPNAME); + if env::args().collect::>().len() == 1 { + return cli.print_help(); + } + + let params = cli.to_owned().get_matches(); + + // Print help? + if *params.get_one::("help").unwrap() { + return cli.print_help(); + } + + // Print version? + if *params.get_one::("version").unwrap() { + println!("Version: {} {}", APPNAME, VERSION); + return Ok(()); + } + + // Setup logger + if let Err(err) = log::set_logger(&LOGGER).map(|()| { + log::set_max_level(match params.get_count("debug") { + 0 => LevelFilter::Info, + 1 => LevelFilter::Debug, + 2.. => LevelFilter::max(), + }) + }) { + println!("Error setting logger output: {}", err); + } + + // Start + if *params.get_one::("start").unwrap_or(&false) { + let cfp = params.get_one::("config"); + if let Err(err) = minion::minion(PathBuf::from(cfp.map_or("", |v| v))).await { + log::error!("Unable to start minion: {}", err); + return Ok(()); + } + } + + Ok(()) +} diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs new file mode 100644 index 00000000..7923c26d --- /dev/null +++ b/sysminion/src/minion.rs @@ -0,0 +1,86 @@ +use libsysinspect::{util, SysinspectError}; +use std::{path::PathBuf, sync::Arc}; +use tokio::net::TcpStream; +use tokio::sync::Mutex; +use tokio::{io::AsyncWriteExt, net::tcp::OwnedWriteHalf}; +use tokio::{ + io::{AsyncReadExt, BufReader}, + sync::mpsc, +}; + +use crate::config; + +/// Talk-back to the master +pub async fn master_feedback(stream: Arc>, msg: Vec) { + let mut stm = stream.lock().await; + + if let Err(e) = stm.write_all(&(msg.len() as u32).to_be_bytes()).await { + log::error!("Failed to send message length to master: {}", e); + return; + } + + if let Err(e) = stm.write_all(&msg).await { + log::error!("Failed to send message to master: {}", e); + return; + } + + if let Err(e) = stm.flush().await { + log::error!("Failed to flush writer to master: {}", e); + } else { + log::debug!("To master: {}", String::from_utf8_lossy(&msg)); + } +} + +/// Minion routine +pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { + if !cfp.exists() { + cfp = util::cfg::select_config()?; + } + let cfg = config::MinionConfig::new(cfp)?; + + let (rstm, wstm) = TcpStream::connect(cfg.master()).await?.into_split(); + let wstm = Arc::new(Mutex::new(wstm)); + let (_w_chan, mut r_chan) = mpsc::channel(100); + + // ehlo + tokio::spawn(master_feedback(wstm.clone(), format!("Connected to {}", cfg.master()).as_bytes().to_vec())); + + // Data exchange + let wtsm_c = wstm.clone(); + tokio::spawn(async move { + let mut input = BufReader::new(rstm); + loop { + let mut buff = [0u8; 4]; + if let Err(e) = input.read_exact(&mut buff).await { + log::error!("Unknown message length from the master: {}", e); + break; + } + let msg_len = u32::from_be_bytes(buff) as usize; + + let mut msg = vec![0u8; msg_len]; + if let Err(e) = input.read_exact(&mut msg).await { + log::error!("Invalid message from the master: {}", e); + break; + } + + log::info!("Received: {}", String::from_utf8_lossy(&msg)); + + // Send a response back to the master after receiving each message + let response = format!("Back: '{}'", String::from_utf8_lossy(&msg)).as_bytes().to_vec(); + master_feedback(wtsm_c.clone(), response).await; + } + }); + + // Task to handle queued messages for the master + let qmsg_stm = wstm.to_owned(); + tokio::spawn(async move { + while let Some(msg) = r_chan.recv().await { + master_feedback(qmsg_stm.clone(), msg).await; + } + }); + + // Keep the client alive until Ctrl+C is pressed + tokio::signal::ctrl_c().await.expect("Failed to listen for ctrl_c"); + log::info!("Shutting down client."); + Ok(()) +} From db1724c7cf36d9d909a95603f7b1e6fd9994f054 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 19:11:50 +0100 Subject: [PATCH 003/148] Add master/minion config --- sysinspect.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sysinspect.conf b/sysinspect.conf index f61790ef..8c1ddcbf 100644 --- a/sysinspect.conf +++ b/sysinspect.conf @@ -1,2 +1,13 @@ config: modules: target/debug + + # Configuration that is present only on master node + master: + # Command socket. Default: /var/run/sysinspect.socket + cmd-socket: /var/run/sysinspect.socket + tcp-port: 4200 + + # Configuration that is present only on a minion node + minion: + master.ip: 192.168.2.102 + master.port: 4200 From dbd619172ac110714ee44713f0452eb92ef56686 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 21:21:22 +0100 Subject: [PATCH 004/148] Add FFI null error handler --- libsysinspect/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 6883283c..189c34d8 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -1,5 +1,6 @@ use std::{ error::Error, + ffi::NulError, fmt::{Display, Formatter, Result}, io, }; @@ -25,6 +26,7 @@ pub enum SysinspectError { IoErr(io::Error), SerdeYaml(serde_yaml::Error), SerdeJson(serde_json::Error), + FFINullError(NulError), } impl Error for SysinspectError { @@ -48,6 +50,7 @@ impl Display for SysinspectError { SysinspectError::ModelDSLError(err) => format!("(DSL) {err}"), SysinspectError::ModuleError(err) => format!("(Module) {err}"), SysinspectError::ConfigError(err) => format!("(Config) {err}"), + SysinspectError::FFINullError(err) => format!("(System) {err}"), }; write!(f, "{msg}")?; @@ -75,3 +78,10 @@ impl From for SysinspectError { SysinspectError::SerdeJson(err) } } + +/// Handle FFI Nul error +impl From for SysinspectError { + fn from(err: NulError) -> Self { + SysinspectError::FFINullError(err) + } +} From eefea5ab22c7df66d9e0d35cb401df6671b6c3ba Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 21:21:41 +0100 Subject: [PATCH 005/148] Add very basic initial master --- sysmaster/Cargo.toml | 31 +++++++++ sysmaster/src/clidef.rs | 74 ++++++++++++++++++++++ sysmaster/src/config.rs | 47 ++++++++++++++ sysmaster/src/main.rs | 70 +++++++++++++++++++++ sysmaster/src/master.rs | 133 +++++++++++++++++++++++++++++++++++++++ sysmaster/src/rmt/mod.rs | 8 +++ 6 files changed, 363 insertions(+) create mode 100644 sysmaster/Cargo.toml create mode 100644 sysmaster/src/clidef.rs create mode 100644 sysmaster/src/config.rs create mode 100644 sysmaster/src/main.rs create mode 100644 sysmaster/src/master.rs create mode 100644 sysmaster/src/rmt/mod.rs diff --git a/sysmaster/Cargo.toml b/sysmaster/Cargo.toml new file mode 100644 index 00000000..25de54cd --- /dev/null +++ b/sysmaster/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "sysmaster" +version = "0.1.0" +edition = "2021" + +[dependencies] +clap = { version = "4.5.20", features = ["unstable-styles"] } +colored = "2.1.0" +ed25519-dalek = { version = "2.1.1", features = [ + "asm", + "batch", + "digest", + "merlin", + "pem", + "pkcs8", + "rand_core", + "signature", + "serde", +] } +futures = "0.3.31" +libc = "0.2.161" +rand = "0.8.5" +rustls = "0.23.16" +rustls-pemfile = "2.2.0" +serde = { version = "1.0.214", features = ["derive"] } +serde_json = "1.0.132" +serde_yaml = "0.9.34" +tokio = { version = "1.41.0", features = ["full"] } +tokio-rustls = "0.26.0" +libsysinspect = { path = "../libsysinspect" } +log = "0.4.22" diff --git a/sysmaster/src/clidef.rs b/sysmaster/src/clidef.rs new file mode 100644 index 00000000..e6af7415 --- /dev/null +++ b/sysmaster/src/clidef.rs @@ -0,0 +1,74 @@ +use clap::builder::styling; +use clap::{Arg, ArgAction, Command}; +use colored::Colorize; + +/// Define CLI arguments and styling +pub fn cli(version: &'static str, appname: &'static str) -> Command { + let styles = styling::Styles::styled() + .header(styling::AnsiColor::Yellow.on_default()) + .usage(styling::AnsiColor::Yellow.on_default()) + .literal(styling::AnsiColor::BrightGreen.on_default()) + .placeholder(styling::AnsiColor::BrightMagenta.on_default()); + + Command::new(appname) + .version(version) + .about(format!("{} - {}", appname.bright_magenta().bold(), "is a master node to operate minion agents")) + .override_usage(format!("{} [OPTIONS]", appname)) + + // Config + .arg( + Arg::new("config") + .short('c') + .long("config") + .help("Alternative path to the config") + ) + .arg( + Arg::new("query") + .short('q') + .long("query") + .help("Target model for operations") + ) + .arg( + Arg::new("start") + .long("start") + .action(ArgAction::SetTrue) + .help("Start master") + ) + + .next_help_heading("Info") + .arg( + Arg::new("status") + .long("status") + .action(ArgAction::SetTrue) + .help("Show connected minions") + ) + + + // Other + .next_help_heading("Other") + .arg( + Arg::new("debug") + .short('d') + .long("debug") + .action(ArgAction::Count) + .help("Set debug mode for more verbose output. Increase this flag for more verbosity."), + ) + .arg( + Arg::new("help") + .short('h') + .long("help") + .action(ArgAction::SetTrue) + .help("Display help"), + ) + .arg( + Arg::new("version") + .short('v') + .long("version") + .action(ArgAction::SetTrue) + .help("Get current version."), + ) + .disable_help_flag(true) // Otherwise it is displayed in a wrong position + .disable_version_flag(true) + .disable_colored_help(false) + .styles(styles) +} diff --git a/sysmaster/src/config.rs b/sysmaster/src/config.rs new file mode 100644 index 00000000..9c142f8c --- /dev/null +++ b/sysmaster/src/config.rs @@ -0,0 +1,47 @@ +use libsysinspect::{intp::functions::get_by_namespace, SysinspectError}; +use serde::{Deserialize, Serialize}; +use serde_yaml::{from_str, from_value, Value}; +use std::{fs, path::PathBuf}; + +static DEFAULT_ADDR: &str = "0.0.0.0"; +static DEFAULT_PORT: u32 = 4200; +static DEFAULT_SOCKET: &str = "/var/run/sysinspect-master.socket"; + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +pub struct MasterConfig { + // Bind IP listener. Default "the world", i.e. 0.0.0.0 + #[serde(rename = "bind.ip")] + bind_ip: Option, + + // Bind port. Default 4200 + #[serde(rename = "bind.port")] + bind_port: Option, + + // Path to FIFO socket. Default: /var/run/sysinspect-master.socket + socket: Option, +} + +impl MasterConfig { + pub fn new(p: PathBuf) -> Result { + let cp = p.as_os_str().to_str().unwrap_or_default(); + if !p.exists() { + return Err(SysinspectError::ConfigError(format!("File not found: {}", cp))); + } + + if let Some(cfgv) = get_by_namespace(Some(from_str::(&fs::read_to_string(&p)?)?), "config.master") { + return Ok(from_value::(cfgv)?); + } + + Err(SysinspectError::ConfigError(format!("Unable to read config at: {}", cp))) + } + + /// Return master addr + pub fn bind_addr(&self) -> String { + format!("{}:{}", self.bind_ip.to_owned().unwrap_or(DEFAULT_ADDR.to_string()), self.bind_port.unwrap_or(DEFAULT_PORT)) + } + + /// Get socket address + pub fn socket(&self) -> String { + self.socket.to_owned().unwrap_or(DEFAULT_SOCKET.to_string()) + } +} diff --git a/sysmaster/src/main.rs b/sysmaster/src/main.rs new file mode 100644 index 00000000..4e105133 --- /dev/null +++ b/sysmaster/src/main.rs @@ -0,0 +1,70 @@ +mod clidef; +mod config; +mod master; +mod rmt; + +use clidef::cli; +use libsysinspect::{logger, util::cfg::select_config, SysinspectError}; +use log::LevelFilter; +use rmt::send_message; +use std::{env, path::PathBuf}; + +static APPNAME: &str = "sysmaster"; +static VERSION: &str = "0.0.1"; +static LOGGER: logger::STDOUTLogger = logger::STDOUTLogger; + +#[tokio::main] +async fn main() -> Result<(), SysinspectError> { + let mut cli = cli(VERSION, APPNAME); + let params = cli.to_owned().get_matches(); + + // Print help? + if env::args().collect::>().len() == 1 || *params.get_one::("help").unwrap() { + cli.print_help()?; + std::process::exit(1); + } + + // Print version? + if *params.get_one::("version").unwrap() { + println!("Version: {} {}", APPNAME, VERSION); + return Ok(()); + } + + // Setup logger + if let Err(err) = log::set_logger(&LOGGER).map(|()| { + log::set_max_level(match params.get_count("debug") { + 0 => LevelFilter::Info, + 1 => LevelFilter::Debug, + 2.. => LevelFilter::max(), + }) + }) { + println!("Error setting logger output: {}", err); + } + + // Get config + let mut cfp = PathBuf::from(params.get_one::("config").unwrap_or(&"".to_string()).to_owned()); + if !cfp.exists() { + cfp = match select_config() { + Ok(cfp) => { + log::debug!("Reading config at {}", cfp.to_str().unwrap_or_default()); + cfp + } + Err(err) => { + log::error!("{}", err); + std::process::exit(1); + } + }; + } + let cfg = config::MasterConfig::new(cfp)?; + + // Mode + let query = params.get_one::("query").unwrap_or(&"".to_string()).to_owned(); + if *params.get_one::("start").unwrap() { + master::master(cfg).await?; + } else if !query.is_empty() { + log::info!("Query: {}", query); + send_message(&query, &cfg.socket()).await? + } + + Ok(()) +} diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs new file mode 100644 index 00000000..0a7c026b --- /dev/null +++ b/sysmaster/src/master.rs @@ -0,0 +1,133 @@ +use crate::config::MasterConfig; +use libsysinspect::SysinspectError; +use std::path::Path; +use tokio::fs::OpenOptions; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as TokioBufReader}; +use tokio::net::TcpListener; +use tokio::select; +use tokio::sync::{broadcast, mpsc}; +use tokio::time::{sleep, Duration}; + +/// Open FIFO socket for command-line communication +fn open_socket(path: &str) -> Result<(), SysinspectError> { + if !Path::new(path).exists() { + if unsafe { libc::mkfifo(std::ffi::CString::new(path)?.as_ptr(), 0o600) } != 0 { + return Err(SysinspectError::ConfigError(format!("{}", std::io::Error::last_os_error()))); + } + log::info!("Socket opened at {}", path); + } + Ok(()) +} + +pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { + log::info!("Starting master at {}", cfg.bind_addr()); + open_socket(&cfg.socket())?; + + let listener = TcpListener::bind(cfg.bind_addr()).await?; + let (tx, _) = broadcast::channel::>(100); + let (client_tx, mut client_rx) = mpsc::channel::<(Vec, usize)>(100); + + // Task to read from the FIFO and broadcast messages to clients + let tx_clone = tx.clone(); + tokio::spawn(async move { + loop { + match OpenOptions::new().read(true).open(cfg.socket()).await { + Ok(file) => { + let reader = TokioBufReader::new(file); + let mut lines = reader.lines(); + + loop { + select! { + line = lines.next_line() => { + match line { + Ok(Some(message)) => { + log::info!("Broadcasting FIFO message to clients: {}", message); + let _ = tx_clone.send(message.into_bytes()); + } + Ok(None) => break, // End of file, re-open the FIFO + Err(e) => { + log::error!("Error reading from FIFO: {}", e); + break; + } + } + } + } + } + } + Err(e) => { + log::error!("Failed to open FIFO: {}", e); + sleep(Duration::from_secs(1)).await; // Retry after a sec + } + } + } + }); + + // Handle incoming messages from minions + tokio::spawn(async move { + loop { + if let Some((msg, client_id)) = client_rx.recv().await { + log::info!("Minion: {}: {}", client_id, String::from_utf8_lossy(&msg)); + } else { + break; + } + } + }); + + // Accept connections and spawn tasks for each client + tokio::spawn(async move { + let mut client_id_counter: usize = 0; + loop { + if let Ok((socket, _)) = listener.accept().await { + client_id_counter += 1; + let current_client_id = client_id_counter; + let mut rx = tx.subscribe(); + let client_tx = client_tx.clone(); + + let (reader, writer) = socket.into_split(); + + // Task to send messages to the client + tokio::spawn(async move { + let mut writer = writer; + log::info!("Minion {} connected. Ready to send messages.", current_client_id); + loop { + if let Ok(msg) = rx.recv().await { + log::info!("Sending message to client {}: {:?}", current_client_id, msg); + if writer.write_all(&(msg.len() as u32).to_be_bytes()).await.is_err() + || writer.write_all(&msg).await.is_err() + || writer.flush().await.is_err() + { + break; + } + } + } + }); + + // Task to read messages from the client + tokio::spawn(async move { + let mut reader = TokioBufReader::new(reader); + loop { + let mut len_buf = [0u8; 4]; + if reader.read_exact(&mut len_buf).await.is_err() { + return; + } + + let msg_len = u32::from_be_bytes(len_buf) as usize; + let mut msg = vec![0u8; msg_len]; + if reader.read_exact(&mut msg).await.is_err() { + return; + } + + if client_tx.send((msg, current_client_id)).await.is_err() { + break; + } + } + }); + } + } + }); + + // Listen for shutdown signal and cancel tasks + tokio::signal::ctrl_c().await.expect("Failed to listen for ctrl_c"); + log::info!("Received shutdown signal."); + std::process::exit(0); +} diff --git a/sysmaster/src/rmt/mod.rs b/sysmaster/src/rmt/mod.rs new file mode 100644 index 00000000..85ab45aa --- /dev/null +++ b/sysmaster/src/rmt/mod.rs @@ -0,0 +1,8 @@ +use libsysinspect::SysinspectError; +use tokio::{fs::OpenOptions, io::AsyncWriteExt}; + +pub(crate) async fn send_message(msg: &str, fifo: &str) -> Result<(), SysinspectError> { + OpenOptions::new().write(true).open(fifo).await?.write_all(format!("{}\n", msg).as_bytes()).await?; + log::debug!("Message sent to FIFO: {}", msg); + Ok(()) +} From 579933f00be0834f80815fa8248b080bd7523f46 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 21:21:57 +0100 Subject: [PATCH 006/148] Lintfix: sort import --- sysminion/src/minion.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 7923c26d..e665781e 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,3 +1,4 @@ +use crate::config; use libsysinspect::{util, SysinspectError}; use std::{path::PathBuf, sync::Arc}; use tokio::net::TcpStream; @@ -8,8 +9,6 @@ use tokio::{ sync::mpsc, }; -use crate::config; - /// Talk-back to the master pub async fn master_feedback(stream: Arc>, msg: Vec) { let mut stm = stream.lock().await; From 8978ccd713678b699e73440289dbf5b50ecfed87 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 21:22:16 +0100 Subject: [PATCH 007/148] Tear down minion after help printed --- sysminion/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index b8a017bb..659d34fd 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -15,7 +15,8 @@ static LOGGER: logger::STDOUTLogger = logger::STDOUTLogger; async fn main() -> std::io::Result<()> { let mut cli = cli(VERSION, APPNAME); if env::args().collect::>().len() == 1 { - return cli.print_help(); + cli.print_help()?; + std::process::exit(1); } let params = cli.to_owned().get_matches(); From c69ea855683a2163227985d3757a7f116966da85 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 21:22:26 +0100 Subject: [PATCH 008/148] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d0f40134..1fecf573 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ docs/_build man/*8 package/ +.tmp/ \ No newline at end of file From 60f2c9bb6c6370a684e3553471716727bd35b4ad Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 21:22:34 +0100 Subject: [PATCH 009/148] Update workspaces and deps --- Cargo.lock | 747 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 748 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 84660d3f..10996e11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,6 +96,33 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-rs" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -111,6 +138,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bincode" version = "1.3.3" @@ -120,6 +153,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.85", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -132,6 +188,15 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -156,9 +221,20 @@ version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -185,6 +261,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.20" @@ -212,6 +299,15 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "cmake" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -228,12 +324,27 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -268,6 +379,16 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "csv" version = "1.3.0" @@ -289,6 +410,54 @@ dependencies = [ "memchr", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -310,6 +479,40 @@ dependencies = [ "winapi", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "serde", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "merlin", + "rand_core", + "serde", + "sha2", + "signature", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.13.0" @@ -338,6 +541,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "flate2" version = "1.0.34" @@ -348,12 +557,117 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fst" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -371,6 +685,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "hashbrown" version = "0.15.0" @@ -395,6 +715,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "hyphenation" version = "0.8.4" @@ -479,12 +808,30 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.72" @@ -494,18 +841,43 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +[[package]] +name = "libloading" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libredox" version = "0.1.3" @@ -579,6 +951,24 @@ dependencies = [ "autocfg", ] +[[package]] +name = "merlin" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d" +dependencies = [ + "byteorder", + "keccak", + "rand_core", + "zeroize", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -600,6 +990,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + [[package]] name = "neli" version = "0.6.4" @@ -653,6 +1049,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" version = "0.4.1" @@ -709,18 +1115,68 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "pin-project-lite" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pocket-resources" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c135f38778ad324d9e9ee68690bac2c1a51f340fdf96ca13e2ab3914eb2e51d8" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn 2.0.85", +] + [[package]] name = "prettytable-rs" version = "0.10.0" @@ -790,6 +1246,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rayon" version = "1.10.0" @@ -859,6 +1345,21 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "run" version = "0.2.0" @@ -876,6 +1377,21 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.27" @@ -903,6 +1419,48 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.23.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.18" @@ -930,6 +1488,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.214" @@ -975,6 +1539,27 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", + "sha2-asm", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", +] + [[package]] name = "shlex" version = "1.3.0" @@ -990,6 +1575,25 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -1012,12 +1616,34 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "1.0.109" @@ -1080,6 +1706,45 @@ dependencies = [ "sysinfo 0.31.4", ] +[[package]] +name = "sysmaster" +version = "0.1.0" +dependencies = [ + "clap", + "colored", + "ed25519-dalek", + "futures", + "libc", + "libsysinspect", + "log", + "rand", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_yaml", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "sysminion" +version = "0.1.0" +dependencies = [ + "clap", + "colored", + "ed25519-dalek", + "libsysinspect", + "log", + "rand", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_yaml", + "tokio", +] + [[package]] name = "term" version = "0.7.0" @@ -1163,6 +1828,23 @@ dependencies = [ "syn 2.0.85", ] +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-ident" version = "1.0.13" @@ -1193,12 +1875,24 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -1270,6 +1964,18 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.38", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1510,3 +2216,44 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] diff --git a/Cargo.toml b/Cargo.toml index 8cad8e47..bf2c4787 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ sysinfo = { version = "0.31.4", features = ["linux-tmpfs"] } [workspace] resolver = "2" -members = ["modules/sys/*", "libsysinspect"] +members = ["modules/sys/*", "libsysinspect", "sysmaster", "sysminion"] [profile.release] strip = true From 1b5546085618178103ddce9e835f2d8c89620960 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 21:22:45 +0100 Subject: [PATCH 010/148] Update general config for the master --- sysinspect.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sysinspect.conf b/sysinspect.conf index 8c1ddcbf..85ad5ede 100644 --- a/sysinspect.conf +++ b/sysinspect.conf @@ -4,8 +4,9 @@ config: # Configuration that is present only on master node master: # Command socket. Default: /var/run/sysinspect.socket - cmd-socket: /var/run/sysinspect.socket - tcp-port: 4200 + socket: /tmp/sysinspect-master.socket + bind.ip: 0.0.0.0 + bind.port: 4200 # Configuration that is present only on a minion node minion: From 634a2aa766430c03ba451a82db0e887d39e25ae0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 2 Nov 2024 22:28:29 +0100 Subject: [PATCH 011/148] Define system traits for a minion --- Cargo.lock | 2 + sysminion/Cargo.toml | 2 + sysminion/src/main.rs | 1 + sysminion/src/minion.rs | 3 +- sysminion/src/traits/mod.rs | 24 ++++++++++++ sysminion/src/traits/systraits.rs | 64 +++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 sysminion/src/traits/mod.rs create mode 100644 sysminion/src/traits/systraits.rs diff --git a/Cargo.lock b/Cargo.lock index 10996e11..45b867de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1734,8 +1734,10 @@ dependencies = [ "clap", "colored", "ed25519-dalek", + "indexmap", "libsysinspect", "log", + "once_cell", "rand", "rustls", "rustls-pemfile", diff --git a/sysminion/Cargo.toml b/sysminion/Cargo.toml index e684649e..da71a7e1 100644 --- a/sysminion/Cargo.toml +++ b/sysminion/Cargo.toml @@ -26,3 +26,5 @@ log = "0.4.22" serde_yaml = "0.9.34" serde = { version = "1.0.214", features = ["derive"] } serde_json = "1.0.132" +indexmap = "2.6.0" +once_cell = "1.20.2" diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 659d34fd..2fb81ac0 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -1,6 +1,7 @@ mod clidef; mod config; mod minion; +mod traits; use clidef::cli; use libsysinspect::logger; diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index e665781e..a47854f7 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,4 +1,4 @@ -use crate::config; +use crate::{config, traits}; use libsysinspect::{util, SysinspectError}; use std::{path::PathBuf, sync::Arc}; use tokio::net::TcpStream; @@ -36,6 +36,7 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { cfp = util::cfg::select_config()?; } let cfg = config::MinionConfig::new(cfp)?; + let st = traits::get_traits(); let (rstm, wstm) = TcpStream::connect(cfg.master()).await?.into_split(); let wstm = Arc::new(Mutex::new(wstm)); diff --git a/sysminion/src/traits/mod.rs b/sysminion/src/traits/mod.rs new file mode 100644 index 00000000..8e74f744 --- /dev/null +++ b/sysminion/src/traits/mod.rs @@ -0,0 +1,24 @@ +mod systraits; + +use once_cell::sync::Lazy; +use std::sync::Mutex; +use systraits::SystemTraits; + +/* +Traits are system properties and attributes on which a minion is running. + +P.S. These are not Rust traits. :-) + */ + +/// System traits instance +static TRAITS: Lazy> = Lazy::new(|| Mutex::new(SystemTraits::new())); + +/// Returns a copy of initialised traits. +pub fn get_traits() -> SystemTraits { + let traits = &TRAITS; + if let Ok(traits) = traits.lock() { + return traits.to_owned(); + } + + SystemTraits::default() +} diff --git a/sysminion/src/traits/systraits.rs b/sysminion/src/traits/systraits.rs new file mode 100644 index 00000000..4e4fd9e0 --- /dev/null +++ b/sysminion/src/traits/systraits.rs @@ -0,0 +1,64 @@ +use indexmap::IndexMap; +use serde_json::Value; + +/// SystemTraits contains a key/value of a system properties. +#[derive(Debug, Clone)] +pub struct SystemTraits { + data: IndexMap, +} + +impl SystemTraits { + pub fn new() -> SystemTraits { + log::debug!("Initialising system traits"); + let mut traits = SystemTraits::default(); + traits.get_system(); + traits.get_defined(); + + traits + } + + /// Put a JSON value into traits structure + pub fn put(&mut self, path: String, data: Value) { + self.data.insert(path, data); + } + + /// Get a trait value in JSON + pub fn get(&self, path: String) -> Option { + self.data.get(&path).cloned() + } + + /// Check if trait is present + pub fn has(&self, path: String) -> bool { + self.get(path).is_some() + } + + /// Check if trait matches the requested value. + pub fn matches(&self, path: String, v: Value) -> bool { + if let Some(t) = self.get(path) { + return t.eq(&v); + } + + false + } + + /// Return known trait items + pub fn items(&self) -> Vec { + self.data.keys().map(|s| s.to_string()).collect::>() + } + + /// Read standard system traits + fn get_system(&self) { + log::debug!("Reading system traits data"); + } + + /// Read defined/configured static traits + fn get_defined(&self) { + log::debug!("Reading custon static traits data") + } +} + +impl Default for SystemTraits { + fn default() -> Self { + Self { data: Default::default() } + } +} From 0548d21cf9f5cfba921177b0dbde8a2801b6ffdb Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 02:02:51 +0100 Subject: [PATCH 012/148] Read all basic traits --- sysminion/src/traits/systraits.rs | 63 ++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/sysminion/src/traits/systraits.rs b/sysminion/src/traits/systraits.rs index 4e4fd9e0..5f911513 100644 --- a/sysminion/src/traits/systraits.rs +++ b/sysminion/src/traits/systraits.rs @@ -1,5 +1,7 @@ +use std::{fs, path::PathBuf}; + use indexmap::IndexMap; -use serde_json::Value; +use serde_json::{json, Value}; /// SystemTraits contains a key/value of a system properties. #[derive(Debug, Clone)] @@ -12,6 +14,7 @@ impl SystemTraits { log::debug!("Initialising system traits"); let mut traits = SystemTraits::default(); traits.get_system(); + traits.get_network(); traits.get_defined(); traits @@ -47,8 +50,64 @@ impl SystemTraits { } /// Read standard system traits - fn get_system(&self) { + fn get_system(&mut self) { log::debug!("Reading system traits data"); + let system = sysinfo::System::new_all(); + + // Common + if let Some(v) = sysinfo::System::host_name() { + self.put("system.hostname".to_string(), json!(v)); + } + + if let Some(v) = sysinfo::System::kernel_version() { + self.put("system.kernel".to_string(), json!(v)); + } + + if let Some(v) = sysinfo::System::os_version() { + self.put("system.os.version".to_string(), json!(v)); + } + + if let Some(v) = sysinfo::System::name() { + self.put("system.os.name".to_string(), json!(v)); + } + + self.put("system.os.distribution".to_string(), json!(sysinfo::System::distribution_id())); + + // Machine Id (not always there) + let mip = PathBuf::from("/etc/machine-id"); + let mut mid = String::default(); + if mip.exists() { + if let Ok(id) = fs::read_to_string(mip) { + mid = id.trim().to_string(); + } + } + self.put("system.id".to_string(), json!(mid)); + + // Memory + self.put("system.mem.total".to_string(), json!(system.total_memory())); + self.put("system.swap.total".to_string(), json!(system.total_swap())); + + // Load CPU data + self.put("system.cpu.total".to_string(), json!(system.cpus().len())); + self.put("system.cpu.brand".to_string(), json!(system.cpus()[0].brand())); + self.put("system.cpu.frequency".to_string(), json!(system.cpus()[0].frequency())); + self.put("system.cpu.vendor".to_string(), json!(system.cpus()[0].vendor_id())); + if let Some(pcrc) = system.physical_core_count() { + self.put("system.cpu.cores".to_string(), json!(pcrc)); + } + } + + /// Load network data + fn get_network(&mut self) { + log::debug!("Reading network traits data"); + let net = sysinfo::Networks::new_with_refreshed_list(); + for (ifs, data) in net.iter() { + self.put(format!("system.net.{}.mac", ifs), json!(data.mac_address().to_string())); + for ipn in data.ip_networks() { + let tp = if ipn.addr.is_ipv4() { "4" } else { "6" }; + self.put(format!("system.net.{}.ipv{}", ifs, tp), json!(ipn.addr.to_string())); + } + } } /// Read defined/configured static traits From 2336f455c8037684d1bd5ef9728baefa59919adc Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 02:02:57 +0100 Subject: [PATCH 013/148] Update deps --- Cargo.lock | 1 + sysminion/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 45b867de..5bbda32d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1744,6 +1744,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sysinfo 0.32.0", "tokio", ] diff --git a/sysminion/Cargo.toml b/sysminion/Cargo.toml index da71a7e1..8bbda79c 100644 --- a/sysminion/Cargo.toml +++ b/sysminion/Cargo.toml @@ -28,3 +28,4 @@ serde = { version = "1.0.214", features = ["derive"] } serde_json = "1.0.132" indexmap = "2.6.0" once_cell = "1.20.2" +sysinfo = { version = "0.32.0", features = ["linux-tmpfs"] } From aab2ca17dc7633a64e6848a6916aacd7e1ee9c5c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 02:04:29 +0100 Subject: [PATCH 014/148] Add lintfixes --- sysmaster/src/master.rs | 1 + sysminion/src/traits/systraits.rs | 8 +------- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 0a7c026b..508b01a7 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -63,6 +63,7 @@ pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { }); // Handle incoming messages from minions + #[allow(clippy::while_let_loop)] tokio::spawn(async move { loop { if let Some((msg, client_id)) = client_rx.recv().await { diff --git a/sysminion/src/traits/systraits.rs b/sysminion/src/traits/systraits.rs index 5f911513..d570daf9 100644 --- a/sysminion/src/traits/systraits.rs +++ b/sysminion/src/traits/systraits.rs @@ -4,7 +4,7 @@ use indexmap::IndexMap; use serde_json::{json, Value}; /// SystemTraits contains a key/value of a system properties. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct SystemTraits { data: IndexMap, } @@ -115,9 +115,3 @@ impl SystemTraits { log::debug!("Reading custon static traits data") } } - -impl Default for SystemTraits { - fn default() -> Self { - Self { data: Default::default() } - } -} From 9c8294a35a9cdc633c290187945144b420ef2104 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 14:40:28 +0100 Subject: [PATCH 015/148] Add RSA keygen and PEM import/export --- libsysinspect/Cargo.toml | 5 +++ libsysinspect/src/lib.rs | 1 + libsysinspect/src/rsa/keys.rs | 60 +++++++++++++++++++++++++++++++++++ libsysinspect/src/rsa/mod.rs | 1 + 4 files changed, 67 insertions(+) create mode 100644 libsysinspect/src/rsa/keys.rs create mode 100644 libsysinspect/src/rsa/mod.rs diff --git a/libsysinspect/Cargo.toml b/libsysinspect/Cargo.toml index 6c2477a4..eaca6395 100644 --- a/libsysinspect/Cargo.toml +++ b/libsysinspect/Cargo.toml @@ -4,17 +4,22 @@ version = "0.2.0" edition = "2021" [dependencies] +base64 = "0.22.1" chrono = "0.4.38" colored = "2.1.0" indexmap = "2.6.0" lazy_static = "1.5.0" log = "0.4.22" nix = { version = "0.29.0", features = ["net", "user"] } +pem = "3.0.4" prettytable-rs = "0.10.0" +rand = "0.8.5" regex = "1.10.6" +rsa = "0.9.6" serde = { version = "1.0.210", features = ["derive"] } serde_json = "1.0.128" serde_yaml = "0.9.34" +sha2 = "0.10.8" textwrap = { version = "0.16.1", features = ["hyphenation", "terminal_size"] } unicode-segmentation = "1.12.0" walkdir = "2.5.0" diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 189c34d8..4f54ec2e 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -12,6 +12,7 @@ pub mod logger; pub mod mdescr; pub mod modlib; pub mod reactor; +pub mod rsa; pub mod util; #[derive(Debug)] diff --git a/libsysinspect/src/rsa/keys.rs b/libsysinspect/src/rsa/keys.rs new file mode 100644 index 00000000..772f463a --- /dev/null +++ b/libsysinspect/src/rsa/keys.rs @@ -0,0 +1,60 @@ +use rand::rngs::OsRng; +use rsa::pkcs1::{DecodeRsaPrivateKey, DecodeRsaPublicKey, EncodeRsaPrivateKey, EncodeRsaPublicKey}; +use rsa::{RsaPrivateKey, RsaPublicKey}; +use std::error::Error; + +/// Default key size. +pub static DEFAULT_KEY_SIZE: usize = 1048; + +/// Generate RSA keys +pub fn keygen(bits: usize) -> Result<(RsaPrivateKey, RsaPublicKey), Box> { + let mut rng = OsRng; + let private_key = RsaPrivateKey::new(&mut rng, bits)?; + let public_key = RsaPublicKey::from(&private_key); + + Ok((private_key, public_key)) +} + +/// Serializes RSA private and public keys to PEM format. +/// +/// # Arguments +/// +/// * `private_key` - A reference to the RSA private key. +/// * `public_key` - A reference to the RSA public key. +/// +/// # Returns +/// +/// A tuple containing the PEM-encoded private and public keys as strings. +pub fn to_pem(private_key: &RsaPrivateKey, public_key: &RsaPublicKey) -> Result<(String, String), Box> { + // Serialize private key to PKCS#1 DER + let private_der = private_key.to_pkcs1_der()?; + let private_pem = pem::encode(&pem::Pem::new("RSA PRIVATE KEY", private_der.as_bytes().to_vec())); + + // Serialize public key to PKCS#1 DER + let public_der = public_key.to_pkcs1_der()?; + let public_pem = pem::encode(&pem::Pem::new("RSA PUBLIC KEY", public_der.as_bytes().to_vec())); + + Ok((private_pem, public_pem)) +} + +/// Deserializes RSA private and public keys from PEM format. +/// +/// # Arguments +/// +/// * `private_pem` - A string slice containing the PEM-encoded private key. +/// * `public_pem` - A string slice containing the PEM-encoded public key. +/// +/// # Returns +/// +/// A tuple containing the deserialized RSA private and public keys. +pub fn from_pem(private_pem: &str, public_pem: &str) -> Result<(RsaPrivateKey, RsaPublicKey), Box> { + // Deserialize private key from PEM + let parsed_private_pem = pem::parse(private_pem)?; + let private_key = RsaPrivateKey::from_pkcs1_der(parsed_private_pem.contents())?; + + // Deserialize public key from PEM + let parsed_public_pem = pem::parse(public_pem)?; + let public_key = RsaPublicKey::from_pkcs1_der(parsed_public_pem.contents())?; + + Ok((private_key, public_key)) +} diff --git a/libsysinspect/src/rsa/mod.rs b/libsysinspect/src/rsa/mod.rs new file mode 100644 index 00000000..703bc087 --- /dev/null +++ b/libsysinspect/src/rsa/mod.rs @@ -0,0 +1 @@ +pub mod keys; From ded53b7b9a5d22739e6d0e2443e61b9b9c9c7456 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 16:45:44 +0100 Subject: [PATCH 016/148] Refactor RSA functions --- libsysinspect/Cargo.toml | 2 +- libsysinspect/src/rsa/keys.rs | 95 +++++++++++++++++++---------------- 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/libsysinspect/Cargo.toml b/libsysinspect/Cargo.toml index eaca6395..e258d6dd 100644 --- a/libsysinspect/Cargo.toml +++ b/libsysinspect/Cargo.toml @@ -15,7 +15,7 @@ pem = "3.0.4" prettytable-rs = "0.10.0" rand = "0.8.5" regex = "1.10.6" -rsa = "0.9.6" +rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } serde = { version = "1.0.210", features = ["derive"] } serde_json = "1.0.128" serde_yaml = "0.9.34" diff --git a/libsysinspect/src/rsa/keys.rs b/libsysinspect/src/rsa/keys.rs index 772f463a..fd950aa5 100644 --- a/libsysinspect/src/rsa/keys.rs +++ b/libsysinspect/src/rsa/keys.rs @@ -1,6 +1,12 @@ use rand::rngs::OsRng; -use rsa::pkcs1::{DecodeRsaPrivateKey, DecodeRsaPublicKey, EncodeRsaPrivateKey, EncodeRsaPublicKey}; -use rsa::{RsaPrivateKey, RsaPublicKey}; +use rsa::{ + pkcs1::{DecodeRsaPrivateKey, DecodeRsaPublicKey, EncodeRsaPrivateKey, EncodeRsaPublicKey}, + pkcs1v15::{Signature, SigningKey, VerifyingKey}, + sha2::{Digest, Sha256}, + signature::SignerMut, + signature::{Keypair, SignatureEncoding, Verifier}, + Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey, +}; use std::error::Error; /// Default key size. @@ -9,52 +15,55 @@ pub static DEFAULT_KEY_SIZE: usize = 1048; /// Generate RSA keys pub fn keygen(bits: usize) -> Result<(RsaPrivateKey, RsaPublicKey), Box> { let mut rng = OsRng; - let private_key = RsaPrivateKey::new(&mut rng, bits)?; - let public_key = RsaPublicKey::from(&private_key); + let prk = RsaPrivateKey::new(&mut rng, bits)?; + let pbk = RsaPublicKey::from(&prk); - Ok((private_key, public_key)) + Ok((prk, pbk)) } /// Serializes RSA private and public keys to PEM format. -/// -/// # Arguments -/// -/// * `private_key` - A reference to the RSA private key. -/// * `public_key` - A reference to the RSA public key. -/// -/// # Returns -/// -/// A tuple containing the PEM-encoded private and public keys as strings. -pub fn to_pem(private_key: &RsaPrivateKey, public_key: &RsaPublicKey) -> Result<(String, String), Box> { - // Serialize private key to PKCS#1 DER - let private_der = private_key.to_pkcs1_der()?; - let private_pem = pem::encode(&pem::Pem::new("RSA PRIVATE KEY", private_der.as_bytes().to_vec())); - - // Serialize public key to PKCS#1 DER - let public_der = public_key.to_pkcs1_der()?; - let public_pem = pem::encode(&pem::Pem::new("RSA PUBLIC KEY", public_der.as_bytes().to_vec())); - - Ok((private_pem, public_pem)) +pub fn to_pem(prk: &RsaPrivateKey, pbk: &RsaPublicKey) -> Result<(String, String), Box> { + Ok(( + pem::encode(&pem::Pem::new("RSA PRIVATE KEY", prk.to_pkcs1_der()?.as_bytes().to_vec())), + pem::encode(&pem::Pem::new("RSA PUBLIC KEY", pbk.to_pkcs1_der()?.as_bytes().to_vec())), + )) } /// Deserializes RSA private and public keys from PEM format. -/// -/// # Arguments -/// -/// * `private_pem` - A string slice containing the PEM-encoded private key. -/// * `public_pem` - A string slice containing the PEM-encoded public key. -/// -/// # Returns -/// -/// A tuple containing the deserialized RSA private and public keys. -pub fn from_pem(private_pem: &str, public_pem: &str) -> Result<(RsaPrivateKey, RsaPublicKey), Box> { - // Deserialize private key from PEM - let parsed_private_pem = pem::parse(private_pem)?; - let private_key = RsaPrivateKey::from_pkcs1_der(parsed_private_pem.contents())?; - - // Deserialize public key from PEM - let parsed_public_pem = pem::parse(public_pem)?; - let public_key = RsaPublicKey::from_pkcs1_der(parsed_public_pem.contents())?; - - Ok((private_key, public_key)) +pub fn from_pem(prk_pem: &str, pbk_pem: &str) -> Result<(RsaPrivateKey, RsaPublicKey), Box> { + Ok(( + RsaPrivateKey::from_pkcs1_der(pem::parse(prk_pem)?.contents())?, + RsaPublicKey::from_pkcs1_der(pem::parse(pbk_pem)?.contents())?, + )) +} + +/// Sign data with the private key +pub fn sign_data(prk: RsaPrivateKey, data: &[u8]) -> Result, Box> { + let mut sk = SigningKey::::new(prk); + let sig = sk.sign(data); + sk.verifying_key().verify(data, &sig)?; + + Ok((&*sig.to_bytes()).to_vec()) +} + +/// Verify signature from the pubic key +pub fn verify_sign(pbk: &RsaPublicKey, data: &[u8], sig: Vec) -> Result> { + Ok(VerifyingKey::::new(pbk.clone()).verify(data, &Signature::try_from(sig.as_slice())?).is_ok()) +} + +/// Get fingerprint of a public key +pub fn get_fingerprint(pbk: &RsaPublicKey) -> Result> { + let mut digest = Sha256::new(); + digest.update(pbk.to_pkcs1_der()?.as_bytes()); + Ok(digest.finalize().iter().map(|byte| format!("{:02x}", byte)).collect()) +} + +// Encrypt data +pub fn encrypt(pbk: RsaPublicKey, data: Vec) -> Result, Box> { + Ok(pbk.encrypt(&mut rand::thread_rng(), Pkcs1v15Encrypt, &data[..])?) +} + +// Decrypt data +pub fn decrypt(prk: RsaPrivateKey, cipher: Vec) -> Result, Box> { + Ok(prk.decrypt(Pkcs1v15Encrypt, &cipher)?) } From 442fa47ed579568dc1f56273d7e38dbacd440d2f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 16:49:07 +0100 Subject: [PATCH 017/148] Update deps --- Cargo.lock | 219 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 5bbda32d..33c6345b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "aes" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -138,6 +149,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -197,6 +214,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -215,6 +241,15 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.1.31" @@ -261,6 +296,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -455,7 +500,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -715,6 +762,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -780,6 +836,16 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -855,6 +921,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "lazycell" @@ -878,6 +947,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + [[package]] name = "libredox" version = "0.1.3" @@ -892,17 +967,22 @@ dependencies = [ name = "libsysinspect" version = "0.2.0" dependencies = [ + "base64", "chrono", "colored", "indexmap", "lazy_static", "log", "nix", + "pem", "prettytable-rs", + "rand", "regex", + "rsa", "serde", "serde_json", "serde_yaml", + "sha2", "textwrap", "unicode-segmentation", "walkdir", @@ -1068,6 +1148,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1075,6 +1192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1121,6 +1239,26 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +dependencies = [ + "base64", + "serde", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1142,6 +1280,32 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -1149,6 +1313,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core", "spki", ] @@ -1360,6 +1526,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "sha1", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "run" version = "0.2.0" @@ -1473,6 +1661,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1488,6 +1685,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + [[package]] name = "semver" version = "1.0.23" @@ -1539,6 +1747,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" From 68533f0960987f90f548d71fed0b621b641eefd3 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 17:28:29 +0100 Subject: [PATCH 018/148] Write keys to a file as PEM --- libsysinspect/src/rsa/keys.rs | 64 ++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/libsysinspect/src/rsa/keys.rs b/libsysinspect/src/rsa/keys.rs index fd950aa5..0a21b243 100644 --- a/libsysinspect/src/rsa/keys.rs +++ b/libsysinspect/src/rsa/keys.rs @@ -7,10 +7,16 @@ use rsa::{ signature::{Keypair, SignatureEncoding, Verifier}, Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey, }; -use std::error::Error; +use std::{error::Error, fs, io, path::PathBuf}; + +use crate::SysinspectError; /// Default key size. pub static DEFAULT_KEY_SIZE: usize = 1048; +pub enum RsaKey { + Prk(RsaPrivateKey), + Pbk(RsaPublicKey), +} /// Generate RSA keys pub fn keygen(bits: usize) -> Result<(RsaPrivateKey, RsaPublicKey), Box> { @@ -22,10 +28,20 @@ pub fn keygen(bits: usize) -> Result<(RsaPrivateKey, RsaPublicKey), Box Result<(String, String), Box> { +pub fn to_pem( + prk: Option<&RsaPrivateKey>, pbk: Option<&RsaPublicKey>, +) -> Result<(Option, Option), Box> { Ok(( - pem::encode(&pem::Pem::new("RSA PRIVATE KEY", prk.to_pkcs1_der()?.as_bytes().to_vec())), - pem::encode(&pem::Pem::new("RSA PUBLIC KEY", pbk.to_pkcs1_der()?.as_bytes().to_vec())), + if prk.is_some() { + Some(pem::encode(&pem::Pem::new("RSA PRIVATE KEY", prk.unwrap().to_pkcs1_der()?.as_bytes().to_vec()))) + } else { + None + }, + if pbk.is_some() { + Some(pem::encode(&pem::Pem::new("RSA PUBLIC KEY", pbk.unwrap().to_pkcs1_der()?.as_bytes().to_vec()))) + } else { + None + }, )) } @@ -43,7 +59,7 @@ pub fn sign_data(prk: RsaPrivateKey, data: &[u8]) -> Result, Box) -> Result Result> { let mut digest = Sha256::new(); digest.update(pbk.to_pkcs1_der()?.as_bytes()); @@ -67,3 +84,40 @@ pub fn encrypt(pbk: RsaPublicKey, data: Vec) -> Result, Box) -> Result, Box> { Ok(prk.decrypt(Pkcs1v15Encrypt, &cipher)?) } + +/// Write private key as a file +pub fn key_to_file(prk: &RsaKey, p: &str, name: &str) -> Result<(), SysinspectError> { + let p = PathBuf::from(p).join(name); + if p.exists() { + return Err(SysinspectError::IoErr(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("File {} already exists", p.to_str().unwrap_or_default()), + ))); + } + + let mut pem = String::default(); + match prk { + RsaKey::Prk(prk) => { + if let Ok((prk_pem, _)) = to_pem(Some(prk), None) { + if let Some(prk_pem) = prk_pem { + pem = prk_pem; + } + } else { + return Err(SysinspectError::IoErr(io::Error::new(io::ErrorKind::InvalidData, "Unable to create PEM key"))); + } + } + RsaKey::Pbk(pbk) => { + if let Ok((_, pbk_pem)) = to_pem(None, Some(pbk)) { + if let Some(pbk_pem) = pbk_pem { + pem = pbk_pem; + } + } else { + return Err(SysinspectError::IoErr(io::Error::new(io::ErrorKind::InvalidData, "Unable to create PEM key"))); + } + } + } + fs::write(&p, pem.as_bytes())?; + log::debug!("Wrote PEM file as {}", p.to_str().unwrap_or_default()); + + Ok(()) +} From d26e1c51a2f11c7fe7b67cad0dfac334943f27b1 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 17:51:38 +0100 Subject: [PATCH 019/148] Add IO to files, refactor --- libsysinspect/src/rsa/keys.rs | 52 +++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/libsysinspect/src/rsa/keys.rs b/libsysinspect/src/rsa/keys.rs index 0a21b243..d73825db 100644 --- a/libsysinspect/src/rsa/keys.rs +++ b/libsysinspect/src/rsa/keys.rs @@ -14,8 +14,8 @@ use crate::SysinspectError; /// Default key size. pub static DEFAULT_KEY_SIZE: usize = 1048; pub enum RsaKey { - Prk(RsaPrivateKey), - Pbk(RsaPublicKey), + Private(RsaPrivateKey), + Public(RsaPublicKey), } /// Generate RSA keys @@ -46,10 +46,20 @@ pub fn to_pem( } /// Deserializes RSA private and public keys from PEM format. -pub fn from_pem(prk_pem: &str, pbk_pem: &str) -> Result<(RsaPrivateKey, RsaPublicKey), Box> { +pub fn from_pem( + prk_pem: Option<&str>, pbk_pem: Option<&str>, +) -> Result<(Option, Option), Box> { Ok(( - RsaPrivateKey::from_pkcs1_der(pem::parse(prk_pem)?.contents())?, - RsaPublicKey::from_pkcs1_der(pem::parse(pbk_pem)?.contents())?, + if prk_pem.is_some() { + Some(RsaPrivateKey::from_pkcs1_der(pem::parse(prk_pem.unwrap_or_default())?.contents())?) + } else { + None + }, + if pbk_pem.is_some() { + Some(RsaPublicKey::from_pkcs1_der(pem::parse(pbk_pem.unwrap_or_default())?.contents())?) + } else { + None + }, )) } @@ -85,7 +95,7 @@ pub fn decrypt(prk: RsaPrivateKey, cipher: Vec) -> Result, Box Result<(), SysinspectError> { let p = PathBuf::from(p).join(name); if p.exists() { @@ -97,7 +107,7 @@ pub fn key_to_file(prk: &RsaKey, p: &str, name: &str) -> Result<(), SysinspectEr let mut pem = String::default(); match prk { - RsaKey::Prk(prk) => { + RsaKey::Private(prk) => { if let Ok((prk_pem, _)) = to_pem(Some(prk), None) { if let Some(prk_pem) = prk_pem { pem = prk_pem; @@ -106,7 +116,7 @@ pub fn key_to_file(prk: &RsaKey, p: &str, name: &str) -> Result<(), SysinspectEr return Err(SysinspectError::IoErr(io::Error::new(io::ErrorKind::InvalidData, "Unable to create PEM key"))); } } - RsaKey::Pbk(pbk) => { + RsaKey::Public(pbk) => { if let Ok((_, pbk_pem)) = to_pem(None, Some(pbk)) { if let Some(pbk_pem) = pbk_pem { pem = pbk_pem; @@ -121,3 +131,29 @@ pub fn key_to_file(prk: &RsaKey, p: &str, name: &str) -> Result<(), SysinspectEr Ok(()) } + +/// Read private or a public key from a file +pub fn key_from_file(p: &str) -> Result, SysinspectError> { + let pth = PathBuf::from(p); + if !pth.exists() { + return Err(SysinspectError::IoErr(io::Error::new(io::ErrorKind::NotFound, format!("File {} not found", p)))); + } + + let data = &fs::read_to_string(pth)?; + + if data.contains("RSA PRIVATE KEY") { + if let Ok((prk, _)) = from_pem(Some(data), None) { + if let Some(prk) = prk { + return Ok(Some(RsaKey::Private(prk))); + } + } + } else { + if let Ok((_, pbk)) = from_pem(None, Some(data)) { + if let Some(pbk) = pbk { + return Ok(Some(RsaKey::Public(pbk))); + } + } + } + + Ok(None) +} From da7c5b914ae199104029e8e403d636f153bc8c9a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 18:26:08 +0100 Subject: [PATCH 020/148] Add unit tests for basic RSA operations --- libsysinspect/tests/rsa_keys.rs | 92 +++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 libsysinspect/tests/rsa_keys.rs diff --git a/libsysinspect/tests/rsa_keys.rs b/libsysinspect/tests/rsa_keys.rs new file mode 100644 index 00000000..3ad3066d --- /dev/null +++ b/libsysinspect/tests/rsa_keys.rs @@ -0,0 +1,92 @@ +#[cfg(test)] +mod rsa_test { + use std::fs; + + use libsysinspect::rsa::keys::{ + key_to_file, keygen, sign_data, to_pem, verify_sign, + RsaKey::{Private, Public}, + DEFAULT_KEY_SIZE, + }; + + #[test] + fn test_keygen() { + let r = keygen(DEFAULT_KEY_SIZE); + assert!(r.is_ok(), "Error generating RSA keys"); + } + + #[test] + fn test_to_pem() { + let (pr, pb) = keygen(DEFAULT_KEY_SIZE).unwrap(); + let r = to_pem(Some(&pr), Some(&pb)); + + assert!(r.is_ok(), "Unable to convert RSA keys to PEM"); + + let (prp, pbp) = r.unwrap(); + + assert!(prp.is_some(), "Unable to convert private PEM key"); + assert!(pbp.is_some(), "Unable to convert public PEM key"); + + let prp = prp.unwrap_or_default(); + let pbp = pbp.unwrap_or_default(); + + assert!(prp.contains(" PRIVATE "), "Not a private key"); + assert!(pbp.contains(" PUBLIC "), "Not a public key"); + } + + #[test] + fn test_sign() { + let (pr, pb) = keygen(DEFAULT_KEY_SIZE).unwrap(); + + let data = "Sysinspect can also configure systems!"; + let sig = sign_data(pr.clone(), data.as_bytes()).unwrap(); + + assert!(!sig.is_empty(), "Sig should not be empty"); + + let r = verify_sign(&pb, data.as_bytes(), sig); + + assert!(r.is_ok(), "Verification failed to proceed"); + assert!(r.unwrap(), "Verification didn't succeed"); + } + + #[test] + fn test_cipher() { + let (pr, pb) = keygen(DEFAULT_KEY_SIZE).unwrap(); + let data = "Sysinspect can also configure systems!"; + let cipher = libsysinspect::rsa::keys::encrypt(pb.to_owned(), data.as_bytes().to_vec()).unwrap(); + + assert!(!cipher.is_empty(), "No cipher found"); + + let rdata = String::from_utf8(libsysinspect::rsa::keys::decrypt(pr.to_owned(), cipher).unwrap()).unwrap_or_default(); + + assert!(rdata.eq(data), "Data wasn't properly decryped"); + } + + #[test] + fn test_to_file() { + _ = fs::remove_file("priv.key.pem"); + _ = fs::remove_file("pub.key.pem"); + + let (pr, pb) = keygen(DEFAULT_KEY_SIZE).unwrap(); + + if let Err(err) = key_to_file(&Private(pr), "", "priv.key.pem") { + assert!(false, "Private key error: {}", err); + }; + + if let Err(err) = key_to_file(&Public(pb), "", "pub.key.pem") { + assert!(false, "Public key error: {}", err); + }; + + match libsysinspect::rsa::keys::key_from_file("priv.key.pem").unwrap().unwrap() { + Private(_) => assert!(true), + Public(_) => assert!(false, "Not a public key"), + } + + match libsysinspect::rsa::keys::key_from_file("pub.key.pem").unwrap().unwrap() { + Private(_) => assert!(false, "Not a private"), + Public(_) => assert!(true), + } + + _ = fs::remove_file("priv.key.pem"); + _ = fs::remove_file("pub.key.pem"); + } +} From dcb14a933f8e3b54d25e72ceb4e7e67d6134cc90 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 18:27:03 +0100 Subject: [PATCH 021/148] Update unit tests --- libsysinspect/tests/rsa_keys.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libsysinspect/tests/rsa_keys.rs b/libsysinspect/tests/rsa_keys.rs index 3ad3066d..ee52e918 100644 --- a/libsysinspect/tests/rsa_keys.rs +++ b/libsysinspect/tests/rsa_keys.rs @@ -1,12 +1,11 @@ #[cfg(test)] mod rsa_test { - use std::fs; - use libsysinspect::rsa::keys::{ - key_to_file, keygen, sign_data, to_pem, verify_sign, + decrypt, encrypt, key_from_file, key_to_file, keygen, sign_data, to_pem, verify_sign, RsaKey::{Private, Public}, DEFAULT_KEY_SIZE, }; + use std::fs; #[test] fn test_keygen() { @@ -52,11 +51,11 @@ mod rsa_test { fn test_cipher() { let (pr, pb) = keygen(DEFAULT_KEY_SIZE).unwrap(); let data = "Sysinspect can also configure systems!"; - let cipher = libsysinspect::rsa::keys::encrypt(pb.to_owned(), data.as_bytes().to_vec()).unwrap(); + let cipher = encrypt(pb.to_owned(), data.as_bytes().to_vec()).unwrap(); assert!(!cipher.is_empty(), "No cipher found"); - let rdata = String::from_utf8(libsysinspect::rsa::keys::decrypt(pr.to_owned(), cipher).unwrap()).unwrap_or_default(); + let rdata = String::from_utf8(decrypt(pr.to_owned(), cipher).unwrap()).unwrap_or_default(); assert!(rdata.eq(data), "Data wasn't properly decryped"); } @@ -76,12 +75,12 @@ mod rsa_test { assert!(false, "Public key error: {}", err); }; - match libsysinspect::rsa::keys::key_from_file("priv.key.pem").unwrap().unwrap() { + match key_from_file("priv.key.pem").unwrap().unwrap() { Private(_) => assert!(true), Public(_) => assert!(false, "Not a public key"), } - match libsysinspect::rsa::keys::key_from_file("pub.key.pem").unwrap().unwrap() { + match key_from_file("pub.key.pem").unwrap().unwrap() { Private(_) => assert!(false, "Not a private"), Public(_) => assert!(true), } From 902d3cffaa650d11dba25fa631984017f0d2485c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 3 Nov 2024 18:31:31 +0100 Subject: [PATCH 022/148] Flatten checks --- libsysinspect/src/rsa/keys.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/libsysinspect/src/rsa/keys.rs b/libsysinspect/src/rsa/keys.rs index d73825db..79a349a4 100644 --- a/libsysinspect/src/rsa/keys.rs +++ b/libsysinspect/src/rsa/keys.rs @@ -13,6 +13,8 @@ use crate::SysinspectError; /// Default key size. pub static DEFAULT_KEY_SIZE: usize = 1048; + +#[allow(clippy::large_enum_variant)] pub enum RsaKey { Private(RsaPrivateKey), Public(RsaPublicKey), @@ -142,17 +144,11 @@ pub fn key_from_file(p: &str) -> Result, SysinspectError> { let data = &fs::read_to_string(pth)?; if data.contains("RSA PRIVATE KEY") { - if let Ok((prk, _)) = from_pem(Some(data), None) { - if let Some(prk) = prk { - return Ok(Some(RsaKey::Private(prk))); - } - } - } else { - if let Ok((_, pbk)) = from_pem(None, Some(data)) { - if let Some(pbk) = pbk { - return Ok(Some(RsaKey::Public(pbk))); - } + if let Ok((Some(prk), _)) = from_pem(Some(data), None) { + return Ok(Some(RsaKey::Private(prk))); } + } else if let Ok((_, Some(pbk))) = from_pem(None, Some(data)) { + return Ok(Some(RsaKey::Public(pbk))); } Ok(None) From cb4da8fcf237cc973435e52dbe2bfcacbfc0fee7 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 17:48:04 +0100 Subject: [PATCH 023/148] Add proto description --- libsysinspect/src/lib.rs | 1 + libsysinspect/src/proto/README.md | 153 ++++++++++++++++++++++++++++++ libsysinspect/src/proto/mod.rs | 0 3 files changed, 154 insertions(+) create mode 100644 libsysinspect/src/proto/README.md create mode 100644 libsysinspect/src/proto/mod.rs diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 4f54ec2e..4d8e3e03 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -11,6 +11,7 @@ pub mod intp; pub mod logger; pub mod mdescr; pub mod modlib; +pub mod proto; pub mod reactor; pub mod rsa; pub mod util; diff --git a/libsysinspect/src/proto/README.md b/libsysinspect/src/proto/README.md new file mode 100644 index 00000000..277ab238 --- /dev/null +++ b/libsysinspect/src/proto/README.md @@ -0,0 +1,153 @@ +# Master/minion protocol + +Protocol description about message exchange between master and a minion. + +## Message Structure + +### Master message structure + +The following is a message structure for a master: + +```json +{ + // Target Destinations + "t": [], + + // Request Type + "r": "", + + // Payload Data in base64 (anything) + "d": "", + + // Return code int + "c": 0, +} +``` + +The following types for "`r`" are available: + +- `add` — Minion registration request. Payload contains an RSA public key of a master. +- `cmd` — A regular command to a minion(s). +- `tr` — Request to return all minion traits for the database sync (payload is empty) +or push new (payload exists). This must be used together with the targeting. +- `rm` — Minion un-registration. + +## Targeting + +Type `t` (target) is a list of target structures. A target structure can target minions +by the following criterias: + +1. Hostnames with UNIX type globbing. E.g.: `web*.com`. +2. Machine Id +3. Traits (any) + +### Target Structure + +```json +{ + // Trait List of traits + "t": [], + + // Minion Id List of minion Ids + "id": [], + + // Hostnames List of minion hostnames + "h": [], +} +``` + +Example targeting by an IPv4 trait, using globbing: + +```json +{ + "t": [ + {"system.net.*.ipv4": "192.168.*"}, + ] +} +``` + +Example targeting by an Id: + +```json +{ + "id": "30006546535e428aba0a0caa6712e225", +} +``` + +Example targeting by hosts, starting their domain names as "web": + +```json +{ + "h": "web*", +} +``` + +Example targeting all minions: + +```json +{ + "h": ["*"], +} +``` + +### Minion message structure + +The following is a message structure for a minion: + +```json +{ + // Id Machine id or pre-generated equivalent if none + "id": "", + + // Request Type + "r": "", + + // Payload Data in base64 (anything) + "d": "", + + // Return code int + "c": 0, +} +``` + +The following request types for "`r`" are available: + +- `add` — Minion registration context. Payload contains nothing. + In this case Master responds with `add` request, containing its RSA public key. + A Minion needs to accept it by a fingerprint. + +- `rsp` — A regular response to any command. +- `ehlo` — Hello notice for a newly connected minion (any). Contains Minion Id RSA cipher. + +## Hello (ehlo) + +This sequence requires no established connection. + +1. Master listens. +2. Minion sends type `ehlo` request with no payload. +3. Master checks if the Id is registered. +4. In case there is no Id registered, Master responds with the error code and kills +the connection. +5. Master responds with a non-zero return code, mapped to a successful connection. + +## Minion Registration Sequence + +This sequence requires no established connection. + +1. Master listens. +2. Minion sends a request type `add` with an empty payload. +3. Master checks if the Id is registered. +4. In case there is an Id registered, Master responds with the error code and kills +the connection, awaiting `ehlo` instead. +5. Master sends type `add` response with RSA public key. +6. Minion accepts the key, storing it and responds with type `ehlo`, containing own +Id within RSA cipher, using Master's public key. +7. Master responds with a non-zero return code, mapped to a successful connection. + +## Minion Call + +This sequence requires established successful connection. + +1. Master broadcasts type `cmd` to all minions with the destination mask. +2. Each minion accepts the message and looks if a target matches it. +3. Each Minion responds back with type `rsp`. diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs new file mode 100644 index 00000000..e69de29b From 4506e6828e65b2400957a4f597ebd1a249a8901b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 17:48:22 +0100 Subject: [PATCH 024/148] Add master and minion error definitions --- libsysinspect/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 4d8e3e03..53fb79f9 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -23,6 +23,8 @@ pub enum SysinspectError { ModelDSLError(String), ModuleError(String), ConfigError(String), + MasterGeneralError(String), + MinionGeneralError(String), // Wrappers for the system errors IoErr(io::Error), @@ -53,6 +55,8 @@ impl Display for SysinspectError { SysinspectError::ModuleError(err) => format!("(Module) {err}"), SysinspectError::ConfigError(err) => format!("(Config) {err}"), SysinspectError::FFINullError(err) => format!("(System) {err}"), + SysinspectError::MasterGeneralError(err) => format!("(Master) {err}"), + SysinspectError::MinionGeneralError(err) => format!("(Minion) {err}"), }; write!(f, "{msg}")?; From d9e33246547d32754f519b53e7d531a3fdfa7777 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:41:11 +0100 Subject: [PATCH 025/148] Update proto description --- libsysinspect/src/proto/README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libsysinspect/src/proto/README.md b/libsysinspect/src/proto/README.md index 277ab238..cdcc8469 100644 --- a/libsysinspect/src/proto/README.md +++ b/libsysinspect/src/proto/README.md @@ -45,8 +45,8 @@ by the following criterias: ```json { - // Trait List of traits - "t": [], + // Trait Targeted traits + "t": {}, // Minion Id List of minion Ids "id": [], @@ -60,9 +60,7 @@ Example targeting by an IPv4 trait, using globbing: ```json { - "t": [ - {"system.net.*.ipv4": "192.168.*"}, - ] + "t": {"system.net.*.ipv4": "192.168.*"}, } ``` From c43ce12aba5d5990834c77a2756c4c32b1a9d53b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:41:18 +0100 Subject: [PATCH 026/148] Define error codes --- libsysinspect/src/proto/errcodes.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 libsysinspect/src/proto/errcodes.rs diff --git a/libsysinspect/src/proto/errcodes.rs b/libsysinspect/src/proto/errcodes.rs new file mode 100644 index 00000000..c79c0b63 --- /dev/null +++ b/libsysinspect/src/proto/errcodes.rs @@ -0,0 +1,16 @@ +pub enum ProtoErrorCode { + /// No code + Undef = 0, + + /// Successfully completed + Success = 1, + + /// General unspecified failure + GeneralFailure = 2, + + /// Minion is not registered + NotRegistered = 3, + + /// Minion is already registered + AlreadyRegistered = 4, +} From 5b922972e052e064fa14bde2f51a48c05d051064 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:41:28 +0100 Subject: [PATCH 027/148] Define request types --- libsysinspect/src/proto/rqtypes.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 libsysinspect/src/proto/rqtypes.rs diff --git a/libsysinspect/src/proto/rqtypes.rs b/libsysinspect/src/proto/rqtypes.rs new file mode 100644 index 00000000..3f8fb3c6 --- /dev/null +++ b/libsysinspect/src/proto/rqtypes.rs @@ -0,0 +1,28 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum RequestType { + /// Minion registration request or context. + #[serde(rename = "add")] + Add, + + /// Minion un-registration request. + #[serde(rename = "rm")] + Remove, + + /// Regular response to any Master command + #[serde(rename = "rsp")] + Response, + + /// Regular command to any Minion + #[serde(rename = "cmd")] + Command, + + /// Request to return all minion traits + #[serde(rename = "tr")] + Traits, + + /// Hello/ehlo + #[serde(rename = "ehlo")] + Ehlo, +} From 3aa843f35ccc3150f824bf986eaff7e640f5e051 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:41:43 +0100 Subject: [PATCH 028/148] Implmement master and minion proto messages --- libsysinspect/src/proto/mod.rs | 103 +++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index e69de29b..e7f26184 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -0,0 +1,103 @@ +pub mod errcodes; +pub mod rqtypes; + +use errcodes::ProtoErrorCode; +use rqtypes::RequestType; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +/// Master message +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MasterMessage { + #[serde(rename = "t")] + target: Vec, + + #[serde(rename = "r")] + request: RequestType, + + #[serde(rename = "d")] + data: String, + + #[serde(rename = "c")] + retcode: usize, +} + +impl MasterMessage { + /// Master message constructor + pub fn new(rtype: RequestType, data: String) -> MasterMessage { + MasterMessage { target: vec![], request: rtype, data, retcode: ProtoErrorCode::Undef as usize } + } + + /// Add a target. + pub fn add_target(&mut self, t: MinionTarget) { + self.target.push(t); + } + + /// Set return code + pub fn set_retcode(&mut self, retcode: ProtoErrorCode) { + self.retcode = retcode as usize; + } +} + +/// Minion message +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MinionMessage { + id: String, + + #[serde(rename = "r")] + request: RequestType, + + #[serde(rename = "d")] + data: String, + + #[serde(rename = "c")] + retcode: usize, +} + +impl MinionMessage { + /// Message constructor + pub fn new(id: String, rtype: RequestType, data: String) -> MinionMessage { + MinionMessage { id, request: rtype, data, retcode: ProtoErrorCode::Undef as usize } + } + + /// Set return code + pub fn set_retcode(&mut self, retcode: ProtoErrorCode) { + self.retcode = retcode as usize; + } +} + +/// Minion target +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MinionTarget { + /// List of minion Ids + id: Vec, + + /// List of a collection of traits + #[serde(rename = "t")] + traits: HashMap, + + #[serde(rename = "h")] + hostnames: Vec, +} + +impl MinionTarget { + pub fn new() -> MinionTarget { + MinionTarget { id: vec![], traits: HashMap::default(), hostnames: vec![] } + } + + /// Add target id + pub fn add_minion_id(&mut self, id: String) { + self.id.push(id); + } + + /// Add targeting trait + pub fn add_trait(&mut self, tid: String, v: Value) { + self.traits.insert(tid, v); + } + + /// Add hostnames + pub fn add_hostname(&mut self, hostname: String) { + self.hostnames.push(hostname); + } +} From 2bb5d3a4d40a1fb4b27f4b8b573cd529af71289f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:42:09 +0100 Subject: [PATCH 029/148] Implement on-disk cache minion registry for traits --- sysmaster/src/registry/mod.rs | 71 +++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 sysmaster/src/registry/mod.rs diff --git a/sysmaster/src/registry/mod.rs b/sysmaster/src/registry/mod.rs new file mode 100644 index 00000000..a576b6a8 --- /dev/null +++ b/sysmaster/src/registry/mod.rs @@ -0,0 +1,71 @@ +/* +Minion registry. It contains minion tasks, traits, location and other data + */ + +pub mod rec; + +use libsysinspect::SysinspectError; +use rec::MinionRecord; +use serde_json::json; +use sled::Db; + +static CFG_DEFAULT_ROOT: &str = "/etc/sysinspect"; +static CFG_DB: &str = "registry"; +static CFG_MINION_KEYS: &str = "minion-keys"; + +pub struct MinionRegistry { + conn: Db, +} + +impl MinionRegistry { + pub fn new(pth: &str) -> Result { + Ok(MinionRegistry { + conn: match sled::open(pth) { + Ok(db) => db, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }, + }) + } + + pub fn add(&mut self, mid: &str, mrec: MinionRecord) -> Result<(), SysinspectError> { + if let Err(err) = self.conn.insert(mid, json!(mrec).to_string().as_bytes().to_vec()) { + return Err(SysinspectError::MasterGeneralError(format!("{err}"))); + } + + Ok(()) + } + + pub fn get(&mut self, mid: &str) -> Result, SysinspectError> { + let data = match self.conn.get(mid) { + Ok(data) => data, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }; + + if let Some(data) = data { + return Ok(Some(match String::from_utf8(data.to_vec()) { + Ok(data) => match serde_json::from_str::(&data) { + Ok(mrec) => mrec, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + })); + } + + Ok(None) + } + + pub fn remove(&mut self, mid: &str) -> Result<(), SysinspectError> { + let contains = match self.conn.contains_key(mid) { + Ok(res) => res, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }; + + if contains { + if let Err(err) = self.conn.remove(mid) { + return Err(SysinspectError::MasterGeneralError(format!("{err}"))); + }; + } + + Ok(()) + } +} From 52b8ba3465dcaa179b69fb412a9ce1b7786d4c2b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:42:20 +0100 Subject: [PATCH 030/148] Add minion record message (todo) --- sysmaster/src/registry/rec.rs | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 sysmaster/src/registry/rec.rs diff --git a/sysmaster/src/registry/rec.rs b/sysmaster/src/registry/rec.rs new file mode 100644 index 00000000..a5ba77f9 --- /dev/null +++ b/sysmaster/src/registry/rec.rs @@ -0,0 +1,4 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +pub struct MinionRecord {} From 025c206e488a38cf01c6eabee54c687de4acfadd Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:42:31 +0100 Subject: [PATCH 031/148] Update dependencies --- Cargo.lock | 85 ++++++++++++++++++++++++++++++++++++++++++-- sysmaster/Cargo.toml | 1 + 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33c6345b..1ea4706f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -604,6 +604,16 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -705,6 +715,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -846,6 +865,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -1210,6 +1238,17 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -1217,7 +1256,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -1228,7 +1281,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.7", "smallvec", "windows-targets 0.52.6", ] @@ -1462,6 +1515,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -1813,6 +1875,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -1942,6 +2020,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sled", "tokio", "tokio-rustls", ] @@ -2031,7 +2110,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", diff --git a/sysmaster/Cargo.toml b/sysmaster/Cargo.toml index 25de54cd..67793754 100644 --- a/sysmaster/Cargo.toml +++ b/sysmaster/Cargo.toml @@ -29,3 +29,4 @@ tokio = { version = "1.41.0", features = ["full"] } tokio-rustls = "0.26.0" libsysinspect = { path = "../libsysinspect" } log = "0.4.22" +sled = "0.34.7" From 464c8673063cb239994ed9e0a64a18a986365ace Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 18:43:45 +0100 Subject: [PATCH 032/148] Add default to MinionTarget --- libsysinspect/src/proto/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index e7f26184..ec6114a4 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -68,7 +68,7 @@ impl MinionMessage { } /// Minion target -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct MinionTarget { /// List of minion Ids id: Vec, @@ -83,7 +83,7 @@ pub struct MinionTarget { impl MinionTarget { pub fn new() -> MinionTarget { - MinionTarget { id: vec![], traits: HashMap::default(), hostnames: vec![] } + MinionTarget::default() } /// Add target id From 616676dcbe8fac6eda46f6c9d864eefc11dd1b66 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 22:09:51 +0100 Subject: [PATCH 033/148] Add error code for unknown --- libsysinspect/src/proto/errcodes.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libsysinspect/src/proto/errcodes.rs b/libsysinspect/src/proto/errcodes.rs index c79c0b63..ef218ed2 100644 --- a/libsysinspect/src/proto/errcodes.rs +++ b/libsysinspect/src/proto/errcodes.rs @@ -1,3 +1,6 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] pub enum ProtoErrorCode { /// No code Undef = 0, @@ -13,4 +16,7 @@ pub enum ProtoErrorCode { /// Minion is already registered AlreadyRegistered = 4, + + /// Unassigned, unknown + Unknown, } From f63e30a24d826cdc46567c69b28f1f00bd9751cb Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 22:10:18 +0100 Subject: [PATCH 034/148] Add MinionMessage and serialisation methods --- libsysinspect/src/proto/mod.rs | 55 +++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index ec6114a4..1cfc42f3 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -1,9 +1,10 @@ pub mod errcodes; pub mod rqtypes; +use crate::SysinspectError; use errcodes::ProtoErrorCode; use rqtypes::RequestType; -use serde::{Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; @@ -65,6 +66,33 @@ impl MinionMessage { pub fn set_retcode(&mut self, retcode: ProtoErrorCode) { self.retcode = retcode as usize; } + + /// Get return code + pub fn get_retcode(&self) -> ProtoErrorCode { + match &self.retcode { + 0 => ProtoErrorCode::Undef, + 1 => ProtoErrorCode::Success, + 2 => ProtoErrorCode::GeneralFailure, + 3 => ProtoErrorCode::NotRegistered, + 4 => ProtoErrorCode::AlreadyRegistered, + _ => ProtoErrorCode::Unknown, + } + } + + /// Request type + pub fn req_type(&self) -> &RequestType { + &self.request + } + + /// Get minion Id + pub fn id(&self) -> &str { + &self.id + } + + /// Get payload + pub fn payload(&self) -> &str { + &self.data + } } /// Minion target @@ -101,3 +129,28 @@ impl MinionTarget { self.hostnames.push(hostname); } } + +pub trait ProtoConversion: Serialize + DeserializeOwned { + fn serialise(&self) -> Result; + fn sendable(&self) -> Result, SysinspectError>; +} + +impl ProtoConversion for T +where + T: Serialize + DeserializeOwned, +{ + /// Serialise self + fn serialise(&self) -> Result { + match serde_json::to_string(self) { + Ok(out) => return Ok(out), + Err(err) => { + return Err(SysinspectError::MinionGeneralError(format!("{err}"))); + } + } + } + + /// Serialise self to bytes + fn sendable(&self) -> Result, SysinspectError> { + Ok(self.serialise()?.as_bytes().to_vec()) + } +} From de51f66f30c61e720707aa61af474990e3e763c8 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 22:10:27 +0100 Subject: [PATCH 035/148] Include registry --- sysmaster/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sysmaster/src/main.rs b/sysmaster/src/main.rs index 4e105133..6f795011 100644 --- a/sysmaster/src/main.rs +++ b/sysmaster/src/main.rs @@ -1,6 +1,7 @@ mod clidef; mod config; mod master; +mod registry; mod rmt; use clidef::cli; From cc901eed66f3e29b52a35b5cebc0660ebb4f48db Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 22:10:43 +0100 Subject: [PATCH 036/148] Accept message types --- sysmaster/src/master.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 508b01a7..d1a1065d 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -1,5 +1,8 @@ use crate::config::MasterConfig; -use libsysinspect::SysinspectError; +use libsysinspect::{ + proto::{self, MinionMessage}, + SysinspectError, +}; use std::path::Path; use tokio::fs::OpenOptions; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as TokioBufReader}; @@ -8,6 +11,20 @@ use tokio::select; use tokio::sync::{broadcast, mpsc}; use tokio::time::{sleep, Duration}; +/// Parse minion request +fn to_request(data: &str) -> Option { + match serde_json::from_str::(&data) { + Ok(request) => { + return Some(request); + } + Err(err) => { + log::error!("Error parse minion response: {err}"); + } + } + + None +} + /// Open FIFO socket for command-line communication fn open_socket(path: &str) -> Result<(), SysinspectError> { if !Path::new(path).exists() { @@ -67,7 +84,24 @@ pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { tokio::spawn(async move { loop { if let Some((msg, client_id)) = client_rx.recv().await { - log::info!("Minion: {}: {}", client_id, String::from_utf8_lossy(&msg)); + let msg = String::from_utf8_lossy(&msg).to_string(); + log::trace!("Minion response: {}: {}", client_id, msg); + if let Some(req) = to_request(&msg) { + match req.req_type() { + proto::rqtypes::RequestType::Add => { + log::info!("Add"); + } + proto::rqtypes::RequestType::Response => { + log::info!("Response"); + } + proto::rqtypes::RequestType::Ehlo => { + log::info!("Ehlo from {}", req.id()); + } + _ => { + log::error!("Minion sends unknown request type"); + } + } + } } else { break; } From 77fdf52316acb4aa1fe3486d08f9f0fcd28fe60f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 22:10:56 +0100 Subject: [PATCH 037/148] Refactor standard traits --- sysminion/src/traits/mod.rs | 16 +++++++++++++ sysminion/src/traits/systraits.rs | 40 +++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/sysminion/src/traits/mod.rs b/sysminion/src/traits/mod.rs index 8e74f744..08a0b7ce 100644 --- a/sysminion/src/traits/mod.rs +++ b/sysminion/src/traits/mod.rs @@ -4,6 +4,22 @@ use once_cell::sync::Lazy; use std::sync::Mutex; use systraits::SystemTraits; +/// Standard Traits +pub static SYS_ID: &str = "system.id"; +pub static SYS_OS_KERNEL: &str = "system.kernel"; +pub static SYS_OS_VERSION: &str = "system.os.version"; +pub static SYS_OS_NAME: &str = "system.os.name"; +pub static SYS_OS_DISTRO: &str = "system.os.distribution"; + +pub static SYS_NET_HOSTNAME: &str = "system.hostname"; +pub static HW_MEM: &str = "hardware.memory"; +pub static HW_SWAP: &str = "hardware.swap"; +pub static HW_CPU_TOTAL: &str = "hardware.cpu.total"; +pub static HW_CPU_BRAND: &str = "hardware.cpu.brand"; +pub static HW_CPU_FREQ: &str = "hardware.cpu.frequency"; +pub static HW_CPU_VENDOR: &str = "hardware.cpu.vendor"; +pub static HW_CPU_CORES: &str = "hardware.cpu.cores"; + /* Traits are system properties and attributes on which a minion is running. diff --git a/sysminion/src/traits/systraits.rs b/sysminion/src/traits/systraits.rs index d570daf9..eb079a2b 100644 --- a/sysminion/src/traits/systraits.rs +++ b/sysminion/src/traits/systraits.rs @@ -1,7 +1,14 @@ use std::{fs, path::PathBuf}; use indexmap::IndexMap; +use once_cell::sync::Lazy; use serde_json::{json, Value}; +use tokio::sync::Mutex; + +use crate::traits::{ + HW_CPU_BRAND, HW_CPU_CORES, HW_CPU_FREQ, HW_CPU_TOTAL, HW_CPU_VENDOR, HW_MEM, HW_SWAP, SYS_ID, SYS_NET_HOSTNAME, + SYS_OS_DISTRO, SYS_OS_KERNEL, SYS_OS_NAME, SYS_OS_VERSION, +}; /// SystemTraits contains a key/value of a system properties. #[derive(Debug, Clone, Default)] @@ -56,22 +63,22 @@ impl SystemTraits { // Common if let Some(v) = sysinfo::System::host_name() { - self.put("system.hostname".to_string(), json!(v)); + self.put(SYS_NET_HOSTNAME.to_string(), json!(v)); } if let Some(v) = sysinfo::System::kernel_version() { - self.put("system.kernel".to_string(), json!(v)); + self.put(SYS_OS_KERNEL.to_string(), json!(v)); } if let Some(v) = sysinfo::System::os_version() { - self.put("system.os.version".to_string(), json!(v)); + self.put(SYS_OS_VERSION.to_string(), json!(v)); } if let Some(v) = sysinfo::System::name() { - self.put("system.os.name".to_string(), json!(v)); + self.put(SYS_OS_NAME.to_string(), json!(v)); } - self.put("system.os.distribution".to_string(), json!(sysinfo::System::distribution_id())); + self.put(SYS_OS_DISTRO.to_string(), json!(sysinfo::System::distribution_id())); // Machine Id (not always there) let mip = PathBuf::from("/etc/machine-id"); @@ -81,19 +88,19 @@ impl SystemTraits { mid = id.trim().to_string(); } } - self.put("system.id".to_string(), json!(mid)); + self.put(SYS_ID.to_string(), json!(mid)); // Memory - self.put("system.mem.total".to_string(), json!(system.total_memory())); - self.put("system.swap.total".to_string(), json!(system.total_swap())); + self.put(HW_MEM.to_string(), json!(system.total_memory())); + self.put(HW_SWAP.to_string(), json!(system.total_swap())); // Load CPU data - self.put("system.cpu.total".to_string(), json!(system.cpus().len())); - self.put("system.cpu.brand".to_string(), json!(system.cpus()[0].brand())); - self.put("system.cpu.frequency".to_string(), json!(system.cpus()[0].frequency())); - self.put("system.cpu.vendor".to_string(), json!(system.cpus()[0].vendor_id())); + self.put(HW_CPU_TOTAL.to_string(), json!(system.cpus().len())); + self.put(HW_CPU_BRAND.to_string(), json!(system.cpus()[0].brand())); + self.put(HW_CPU_FREQ.to_string(), json!(system.cpus()[0].frequency())); + self.put(HW_CPU_VENDOR.to_string(), json!(system.cpus()[0].vendor_id())); if let Some(pcrc) = system.physical_core_count() { - self.put("system.cpu.cores".to_string(), json!(pcrc)); + self.put(HW_CPU_CORES.to_string(), json!(pcrc)); } } @@ -115,3 +122,10 @@ impl SystemTraits { log::debug!("Reading custon static traits data") } } + +static _INSTANCE: Lazy> = Lazy::new(|| Mutex::new(SystemTraits::new())); + +/// Get traits +pub async fn get_traits() -> &'static Mutex { + &_INSTANCE +} From 5331bfef2ee5c48c568e1e1f2666b9d3814daea0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 22:11:05 +0100 Subject: [PATCH 038/148] Send ehlo from minion --- sysminion/src/minion.rs | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index a47854f7..a516ac19 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,5 +1,12 @@ -use crate::{config, traits}; -use libsysinspect::{util, SysinspectError}; +use crate::{ + config::{self, MinionConfig}, + traits, +}; +use libsysinspect::{ + proto::{rqtypes::RequestType, MinionMessage, ProtoConversion}, + util::{self, dataconv}, + SysinspectError, +}; use std::{path::PathBuf, sync::Arc}; use tokio::net::TcpStream; use tokio::sync::Mutex; @@ -9,8 +16,20 @@ use tokio::{ sync::mpsc, }; +/// Send ehlo +async fn send_ehlo(stream: Arc>, cfg: MinionConfig) -> Result<(), SysinspectError> { + let r = MinionMessage::new( + dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), + RequestType::Ehlo, + "".to_string(), + ); + + log::info!("Ehlo on {}", cfg.master()); + Ok(request(stream, r.sendable()?).await) +} + /// Talk-back to the master -pub async fn master_feedback(stream: Arc>, msg: Vec) { +pub async fn request(stream: Arc>, msg: Vec) { let mut stm = stream.lock().await; if let Err(e) = stm.write_all(&(msg.len() as u32).to_be_bytes()).await { @@ -36,14 +55,14 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { cfp = util::cfg::select_config()?; } let cfg = config::MinionConfig::new(cfp)?; - let st = traits::get_traits(); + //let st = traits::get_traits(); let (rstm, wstm) = TcpStream::connect(cfg.master()).await?.into_split(); - let wstm = Arc::new(Mutex::new(wstm)); + let wstm: Arc> = Arc::new(Mutex::new(wstm)); let (_w_chan, mut r_chan) = mpsc::channel(100); // ehlo - tokio::spawn(master_feedback(wstm.clone(), format!("Connected to {}", cfg.master()).as_bytes().to_vec())); + send_ehlo(wstm.clone(), cfg).await?; // Data exchange let wtsm_c = wstm.clone(); @@ -67,7 +86,7 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { // Send a response back to the master after receiving each message let response = format!("Back: '{}'", String::from_utf8_lossy(&msg)).as_bytes().to_vec(); - master_feedback(wtsm_c.clone(), response).await; + request(wtsm_c.clone(), response).await; } }); @@ -75,7 +94,7 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { let qmsg_stm = wstm.to_owned(); tokio::spawn(async move { while let Some(msg) = r_chan.recv().await { - master_feedback(qmsg_stm.clone(), msg).await; + request(qmsg_stm.clone(), msg).await; } }); From 988b8696a868e35cbad9356d15e8b51e7baa8d54 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 4 Nov 2024 22:13:15 +0100 Subject: [PATCH 039/148] Refactor with lintfixes --- libsysinspect/src/proto/mod.rs | 6 ++---- sysmaster/src/master.rs | 2 +- sysminion/src/minion.rs | 3 ++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 1cfc42f3..41367887 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -142,10 +142,8 @@ where /// Serialise self fn serialise(&self) -> Result { match serde_json::to_string(self) { - Ok(out) => return Ok(out), - Err(err) => { - return Err(SysinspectError::MinionGeneralError(format!("{err}"))); - } + Ok(out) => Ok(out), + Err(err) => Err(SysinspectError::MinionGeneralError(format!("{err}"))), } } diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index d1a1065d..e82d0f9c 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -13,7 +13,7 @@ use tokio::time::{sleep, Duration}; /// Parse minion request fn to_request(data: &str) -> Option { - match serde_json::from_str::(&data) { + match serde_json::from_str::(data) { Ok(request) => { return Some(request); } diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index a516ac19..c515c90a 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -25,7 +25,8 @@ async fn send_ehlo(stream: Arc>, cfg: MinionConfig) -> Res ); log::info!("Ehlo on {}", cfg.master()); - Ok(request(stream, r.sendable()?).await) + request(stream, r.sendable()?).await; + Ok(()) } /// Talk-back to the master From 44706347d560fc8aa31005699ccb0a3586c430fe Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 5 Nov 2024 22:39:11 +0100 Subject: [PATCH 040/148] Add minion key registry --- sysmaster/src/registry/mkb.rs | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 sysmaster/src/registry/mkb.rs diff --git a/sysmaster/src/registry/mkb.rs b/sysmaster/src/registry/mkb.rs new file mode 100644 index 00000000..70d34e35 --- /dev/null +++ b/sysmaster/src/registry/mkb.rs @@ -0,0 +1,42 @@ +use super::{CFG_DEFAULT_ROOT, CFG_MINION_KEYS}; +use libsysinspect::SysinspectError; +use std::{collections::HashMap, fs, path::PathBuf}; + +/// Registered minion base. +/// Essentially this is just a directory, +/// where collected all public keys from all minions. + +#[derive(Debug, Default, Clone)] +pub struct MinionKeyRegistry { + root: PathBuf, + keys: HashMap, +} + +impl MinionKeyRegistry { + pub fn new() -> Result { + let mut reg = + MinionKeyRegistry { root: PathBuf::from(CFG_DEFAULT_ROOT).join(CFG_MINION_KEYS), ..MinionKeyRegistry::default() }; + reg.setup()?; + + Ok(reg) + } + + /// Sets up the registry + fn setup(&mut self) -> Result<(), SysinspectError> { + if !self.root.exists() { + fs::create_dir_all(&self.root)?; + } else { + for e in fs::read_dir(&self.root)?.flatten() { + self.keys + .insert(e.file_name().to_str().and_then(|e| e.split('.').next()).unwrap_or_default().to_string(), e.path()); + } + } + + Ok(()) + } + + /// Returns a method if a minion Id is known to the key registry. + pub fn is_registered(&self, mid: String) -> bool { + self.keys.contains_key(&mid) + } +} From c1b51d510204ae79466a2d27a0ff08abad76cf13 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 5 Nov 2024 22:39:22 +0100 Subject: [PATCH 041/148] Make defaults public --- sysmaster/src/registry/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sysmaster/src/registry/mod.rs b/sysmaster/src/registry/mod.rs index a576b6a8..497412e2 100644 --- a/sysmaster/src/registry/mod.rs +++ b/sysmaster/src/registry/mod.rs @@ -2,6 +2,7 @@ Minion registry. It contains minion tasks, traits, location and other data */ +pub mod mkb; pub mod rec; use libsysinspect::SysinspectError; @@ -9,9 +10,9 @@ use rec::MinionRecord; use serde_json::json; use sled::Db; -static CFG_DEFAULT_ROOT: &str = "/etc/sysinspect"; -static CFG_DB: &str = "registry"; -static CFG_MINION_KEYS: &str = "minion-keys"; +pub static CFG_DEFAULT_ROOT: &str = "/etc/sysinspect"; +pub static CFG_DB: &str = "registry"; +pub static CFG_MINION_KEYS: &str = "minion-keys"; pub struct MinionRegistry { conn: Db, From 8485facc913caf29b2abd99c75a0820190ceabff Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 5 Nov 2024 22:39:30 +0100 Subject: [PATCH 042/148] Refactor master --- sysmaster/src/master.rs | 306 ++++++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 125 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index e82d0f9c..7b88e400 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -3,163 +3,219 @@ use libsysinspect::{ proto::{self, MinionMessage}, SysinspectError, }; -use std::path::Path; -use tokio::fs::OpenOptions; +use std::{path::Path, sync::Arc}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as TokioBufReader}; use tokio::net::TcpListener; use tokio::select; use tokio::sync::{broadcast, mpsc}; use tokio::time::{sleep, Duration}; +use tokio::{fs::OpenOptions, sync::Mutex}; -/// Parse minion request -fn to_request(data: &str) -> Option { - match serde_json::from_str::(data) { - Ok(request) => { - return Some(request); +#[derive(Debug)] +pub struct SysMaster { + cfg: MasterConfig, + broadcast: broadcast::Sender>, +} + +impl SysMaster { + pub fn new(cfg: MasterConfig) -> SysMaster { + let (tx, _) = broadcast::channel::>(100); + + SysMaster { cfg, broadcast: tx } + } + + /// Open FIFO socket for command-line communication + fn open_socket(&self, path: &str) -> Result<(), SysinspectError> { + if !Path::new(path).exists() { + if unsafe { libc::mkfifo(std::ffi::CString::new(path)?.as_ptr(), 0o600) } != 0 { + return Err(SysinspectError::ConfigError(format!("{}", std::io::Error::last_os_error()))); + } + log::info!("Socket opened at {}", path); } - Err(err) => { - log::error!("Error parse minion response: {err}"); + Ok(()) + } + + /// Parse minion request + fn to_request(&self, data: &str) -> Option { + match serde_json::from_str::(data) { + Ok(request) => { + return Some(request); + } + Err(err) => { + log::error!("Error parse minion response: {err}"); + } } + + None } - None -} + /// Start sysmaster + pub async fn init(&mut self) -> Result<(), SysinspectError> { + log::info!("Starting master at {}", self.cfg.bind_addr()); + self.open_socket(&self.cfg.socket())?; + Ok(()) + } -/// Open FIFO socket for command-line communication -fn open_socket(path: &str) -> Result<(), SysinspectError> { - if !Path::new(path).exists() { - if unsafe { libc::mkfifo(std::ffi::CString::new(path)?.as_ptr(), 0o600) } != 0 { - return Err(SysinspectError::ConfigError(format!("{}", std::io::Error::last_os_error()))); - } - log::info!("Socket opened at {}", path); + pub fn cfg(&self) -> MasterConfig { + self.cfg.to_owned() } - Ok(()) -} -pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { - log::info!("Starting master at {}", cfg.bind_addr()); - open_socket(&cfg.socket())?; + pub fn broadcast(&self) -> broadcast::Sender> { + self.broadcast.clone() + } - let listener = TcpListener::bind(cfg.bind_addr()).await?; - let (tx, _) = broadcast::channel::>(100); - let (client_tx, mut client_rx) = mpsc::channel::<(Vec, usize)>(100); + pub async fn listener(&self) -> Result { + Ok(TcpListener::bind(self.cfg.bind_addr()).await?) + } - // Task to read from the FIFO and broadcast messages to clients - let tx_clone = tx.clone(); - tokio::spawn(async move { - loop { - match OpenOptions::new().read(true).open(cfg.socket()).await { - Ok(file) => { - let reader = TokioBufReader::new(file); - let mut lines = reader.lines(); - - loop { - select! { - line = lines.next_line() => { - match line { - Ok(Some(message)) => { - log::info!("Broadcasting FIFO message to clients: {}", message); - let _ = tx_clone.send(message.into_bytes()); - } - Ok(None) => break, // End of file, re-open the FIFO - Err(e) => { - log::error!("Error reading from FIFO: {}", e); - break; - } - } + /// Process incoming minion messages + pub async fn do_incoming(master: Arc>, mut rx: tokio::sync::mpsc::Receiver<(Vec, usize)>) { + log::trace!("Init incoming channel"); + tokio::spawn(async move { + loop { + if let Some((msg, client_id)) = rx.recv().await { + let msg = String::from_utf8_lossy(&msg).to_string(); + log::trace!("Minion response: {}: {}", client_id, msg); + if let Some(req) = master.lock().await.to_request(&msg) { + match req.req_type() { + proto::rqtypes::RequestType::Add => { + log::info!("Add"); + } + proto::rqtypes::RequestType::Response => { + log::info!("Response"); + } + proto::rqtypes::RequestType::Ehlo => { + log::info!("Ehlo from {}", req.id()); + } + _ => { + log::error!("Minion sends unknown request type"); } } } - } - Err(e) => { - log::error!("Failed to open FIFO: {}", e); - sleep(Duration::from_secs(1)).await; // Retry after a sec + } else { + break; } } - } - }); + }); + } - // Handle incoming messages from minions - #[allow(clippy::while_let_loop)] - tokio::spawn(async move { - loop { - if let Some((msg, client_id)) = client_rx.recv().await { - let msg = String::from_utf8_lossy(&msg).to_string(); - log::trace!("Minion response: {}: {}", client_id, msg); - if let Some(req) = to_request(&msg) { - match req.req_type() { - proto::rqtypes::RequestType::Add => { - log::info!("Add"); - } - proto::rqtypes::RequestType::Response => { - log::info!("Response"); - } - proto::rqtypes::RequestType::Ehlo => { - log::info!("Ehlo from {}", req.id()); - } - _ => { - log::error!("Minion sends unknown request type"); + pub async fn do_fifo(master: Arc>) { + log::trace!("Init local command channel"); + tokio::spawn(async move { + let bcast = master.lock().await.broadcast(); + let cfg = master.lock().await.cfg(); + loop { + match OpenOptions::new().read(true).open(cfg.socket()).await { + Ok(file) => { + let reader = TokioBufReader::new(file); + let mut lines = reader.lines(); + + loop { + select! { + line = lines.next_line() => { + match line { + Ok(Some(message)) => { + log::info!("Broadcasting FIFO message to clients: {}", message); + let _ = bcast.send(message.into_bytes()); + } + Ok(None) => break, // End of file, re-open the FIFO + Err(e) => { + log::error!("Error reading from FIFO: {}", e); + break; + } + } + } + } } } + Err(e) => { + log::error!("Failed to open FIFO: {}", e); + sleep(Duration::from_secs(1)).await; // Retry after a sec + } } - } else { - break; } - } - }); + }); + } - // Accept connections and spawn tasks for each client - tokio::spawn(async move { - let mut client_id_counter: usize = 0; - loop { - if let Ok((socket, _)) = listener.accept().await { - client_id_counter += 1; - let current_client_id = client_id_counter; - let mut rx = tx.subscribe(); - let client_tx = client_tx.clone(); - - let (reader, writer) = socket.into_split(); - - // Task to send messages to the client - tokio::spawn(async move { - let mut writer = writer; - log::info!("Minion {} connected. Ready to send messages.", current_client_id); - loop { - if let Ok(msg) = rx.recv().await { - log::info!("Sending message to client {}: {:?}", current_client_id, msg); - if writer.write_all(&(msg.len() as u32).to_be_bytes()).await.is_err() - || writer.write_all(&msg).await.is_err() - || writer.flush().await.is_err() - { - break; + pub async fn do_outgoing( + master: Arc>, tx: tokio::sync::mpsc::Sender<(Vec, usize)>, + ) -> Result<(), SysinspectError> { + log::trace!("Init outgoing channel"); + let listener = master.lock().await.listener().await?; + tokio::spawn(async move { + let bcast = master.lock().await.broadcast(); + let mut client_id_counter: usize = 0; + loop { + if let Ok((socket, _)) = listener.accept().await { + client_id_counter += 1; + let current_client_id = client_id_counter; + let mut rx = bcast.subscribe(); + let client_tx = tx.clone(); + + let (reader, writer) = socket.into_split(); + + // Task to send messages to the client + tokio::spawn(async move { + let mut writer = writer; + log::info!("Minion {} connected. Ready to send messages.", current_client_id); + loop { + if let Ok(msg) = rx.recv().await { + log::info!("Sending message to client {}: {:?}", current_client_id, msg); + if writer.write_all(&(msg.len() as u32).to_be_bytes()).await.is_err() + || writer.write_all(&msg).await.is_err() + || writer.flush().await.is_err() + { + break; + } } } - } - }); - - // Task to read messages from the client - tokio::spawn(async move { - let mut reader = TokioBufReader::new(reader); - loop { - let mut len_buf = [0u8; 4]; - if reader.read_exact(&mut len_buf).await.is_err() { - return; - } + }); - let msg_len = u32::from_be_bytes(len_buf) as usize; - let mut msg = vec![0u8; msg_len]; - if reader.read_exact(&mut msg).await.is_err() { - return; - } + // Task to read messages from the client + tokio::spawn(async move { + let mut reader = TokioBufReader::new(reader); + loop { + let mut len_buf = [0u8; 4]; + if reader.read_exact(&mut len_buf).await.is_err() { + return; + } + + let msg_len = u32::from_be_bytes(len_buf) as usize; + let mut msg = vec![0u8; msg_len]; + if reader.read_exact(&mut msg).await.is_err() { + return; + } - if client_tx.send((msg, current_client_id)).await.is_err() { - break; + if client_tx.send((msg, current_client_id)).await.is_err() { + break; + } } - } - }); + }); + } } - } - }); + }); + + Ok(()) + } +} + +pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { + let master = Arc::new(Mutex::new(SysMaster::new(cfg))); + { + let mut m = master.lock().await; + m.init().await?; + } + + let (client_tx, client_rx) = mpsc::channel::<(Vec, usize)>(100); + + // Task to read from the FIFO and broadcast messages to clients + SysMaster::do_fifo(Arc::clone(&master)).await; + + // Handle incoming messages from minions + SysMaster::do_incoming(Arc::clone(&master), client_rx).await; + + // Accept connections and spawn tasks for each client + SysMaster::do_outgoing(Arc::clone(&master), client_tx).await?; // Listen for shutdown signal and cancel tasks tokio::signal::ctrl_c().await.expect("Failed to listen for ctrl_c"); From 7c20373db17d5d287b578ea9cc244bd79261ecf4 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 5 Nov 2024 22:40:11 +0100 Subject: [PATCH 043/148] Lintfix --- sysmaster/src/master.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 7b88e400..20e67089 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -69,6 +69,7 @@ impl SysMaster { } /// Process incoming minion messages + #[allow(clippy::while_let_loop)] pub async fn do_incoming(master: Arc>, mut rx: tokio::sync::mpsc::Receiver<(Vec, usize)>) { log::trace!("Init incoming channel"); tokio::spawn(async move { From b6b98e3d79b6d3c3b77150ea091763b6354cd3b8 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 18:10:01 +0100 Subject: [PATCH 044/148] Start proto parser for Minion --- sysminion/src/proto.rs | 44 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 sysminion/src/proto.rs diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs new file mode 100644 index 00000000..5a5024a2 --- /dev/null +++ b/sysminion/src/proto.rs @@ -0,0 +1,44 @@ +pub mod msg { + use crate::{config::MinionConfig, minion::request, traits}; + use libsysinspect::{ + proto::{rqtypes::RequestType, MinionMessage}, + util::dataconv, + }; + use libsysinspect::{ + proto::{MasterMessage, ProtoConversion}, + SysinspectError, + }; + use std::sync::Arc; + use tokio::{net::tcp::OwnedWriteHalf, sync::Mutex}; + + /// Send ehlo + pub async fn send_ehlo(stream: Arc>, cfg: MinionConfig) -> Result<(), SysinspectError> { + let r = MinionMessage::new( + dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), + RequestType::Ehlo, + "".to_string(), + ); + + log::info!("Ehlo on {}", cfg.master()); + request(stream, r.sendable()?).await; + Ok(()) + } + + /// Get message + pub fn get_message(data: Vec) -> Result { + let data = match String::from_utf8(data) { + Ok(data) => data, + Err(err) => return Err(SysinspectError::ProtoError(format!("unable to parse master message: {err}"))), + }; + + let msg = match serde_json::from_str::(&data) { + Ok(msg) => msg, + Err(err) => { + log::trace!("Broken JSON message: {data}"); + return Err(SysinspectError::ProtoError(format!("broken JSON from master message: {err}"))); + } + }; + + Ok(msg) + } +} From 460faaf045969b1541b52414ed2b08e319e08626 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 18:10:12 +0100 Subject: [PATCH 045/148] Add ProtoError type --- libsysinspect/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 53fb79f9..fa234e00 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -25,6 +25,7 @@ pub enum SysinspectError { ConfigError(String), MasterGeneralError(String), MinionGeneralError(String), + ProtoError(String), // Wrappers for the system errors IoErr(io::Error), @@ -57,6 +58,7 @@ impl Display for SysinspectError { SysinspectError::FFINullError(err) => format!("(System) {err}"), SysinspectError::MasterGeneralError(err) => format!("(Master) {err}"), SysinspectError::MinionGeneralError(err) => format!("(Minion) {err}"), + SysinspectError::ProtoError(err) => format!("(Protocol) {err}"), }; write!(f, "{msg}")?; From 122eb4d21cdd63c97c3f9fa8b48fcdafd902c965 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 18:10:32 +0100 Subject: [PATCH 046/148] Add methods for MasterMessage --- libsysinspect/src/proto/mod.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 41367887..119d7c6a 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -39,6 +39,28 @@ impl MasterMessage { pub fn set_retcode(&mut self, retcode: ProtoErrorCode) { self.retcode = retcode as usize; } + + /// Get return code + pub fn get_retcode(&self) -> ProtoErrorCode { + match &self.retcode { + 0 => ProtoErrorCode::Undef, + 1 => ProtoErrorCode::Success, + 2 => ProtoErrorCode::GeneralFailure, + 3 => ProtoErrorCode::NotRegistered, + 4 => ProtoErrorCode::AlreadyRegistered, + _ => ProtoErrorCode::Unknown, + } + } + + /// Request type + pub fn req_type(&self) -> &RequestType { + &self.request + } + + /// Get payload + pub fn payload(&self) -> &str { + &self.data + } } /// Minion message From e05b32a7f7f8da162b5bff586a93038e6d1df45e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 18:11:00 +0100 Subject: [PATCH 047/148] Add another RequestType for unknown minion (unregistered, banned, blocked etc) --- libsysinspect/src/proto/rqtypes.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libsysinspect/src/proto/rqtypes.rs b/libsysinspect/src/proto/rqtypes.rs index 3f8fb3c6..82e4abb4 100644 --- a/libsysinspect/src/proto/rqtypes.rs +++ b/libsysinspect/src/proto/rqtypes.rs @@ -25,4 +25,8 @@ pub enum RequestType { /// Hello/ehlo #[serde(rename = "ehlo")] Ehlo, + + /// Unknown agent + #[serde(rename = "undef")] + AgentUnknown, } From d9742f9c4cc7ffcc01710f6046ff693b6c2b3726 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 18:11:15 +0100 Subject: [PATCH 048/148] Refactor master --- sysmaster/src/master.rs | 172 +++++++++++++++++++++++++++------------- 1 file changed, 116 insertions(+), 56 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 20e67089..78707f5a 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -1,9 +1,10 @@ -use crate::config::MasterConfig; +use crate::{config::MasterConfig, registry::mkb::MinionKeyRegistry}; use libsysinspect::{ - proto::{self, MinionMessage}, + proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; -use std::{path::Path, sync::Arc}; +use rustls::crypto::hash::Hash; +use std::{collections::HashSet, path::Path, sync::Arc}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as TokioBufReader}; use tokio::net::TcpListener; use tokio::select; @@ -15,13 +16,16 @@ use tokio::{fs::OpenOptions, sync::Mutex}; pub struct SysMaster { cfg: MasterConfig, broadcast: broadcast::Sender>, + mkr: MinionKeyRegistry, + to_drop: HashSet, } impl SysMaster { - pub fn new(cfg: MasterConfig) -> SysMaster { + pub fn new(cfg: MasterConfig) -> Result { let (tx, _) = broadcast::channel::>(100); - SysMaster { cfg, broadcast: tx } + let mkr = MinionKeyRegistry::new()?; + Ok(SysMaster { cfg, broadcast: tx, mkr, to_drop: HashSet::default() }) } /// Open FIFO socket for command-line communication @@ -68,25 +72,57 @@ impl SysMaster { Ok(TcpListener::bind(self.cfg.bind_addr()).await?) } + /// Get Minion key registry + fn mkr(&self) -> &MinionKeyRegistry { + &self.mkr + } + + /// Bounce message + fn msg_not_registered(&self, mid: String) -> MasterMessage { + let mut m = MasterMessage::new(RequestType::AgentUnknown, "Minion is not registered".to_string()); + let mut tgt = MinionTarget::new(); + tgt.add_minion_id(mid); + m.add_target(tgt); + m.set_retcode(ProtoErrorCode::NotRegistered); + + m + } + /// Process incoming minion messages #[allow(clippy::while_let_loop)] - pub async fn do_incoming(master: Arc>, mut rx: tokio::sync::mpsc::Receiver<(Vec, usize)>) { + pub async fn do_incoming(master: Arc>, mut rx: tokio::sync::mpsc::Receiver<(Vec, String)>) { log::trace!("Init incoming channel"); + let bcast = master.lock().await.broadcast(); tokio::spawn(async move { loop { - if let Some((msg, client_id)) = rx.recv().await { + if let Some((msg, minion_addr)) = rx.recv().await { let msg = String::from_utf8_lossy(&msg).to_string(); - log::trace!("Minion response: {}: {}", client_id, msg); + log::trace!("Minion response: {}: {}", minion_addr, msg); if let Some(req) = master.lock().await.to_request(&msg) { match req.req_type() { - proto::rqtypes::RequestType::Add => { + RequestType::Add => { log::info!("Add"); } - proto::rqtypes::RequestType::Response => { + RequestType::Response => { log::info!("Response"); } - proto::rqtypes::RequestType::Ehlo => { + RequestType::Ehlo => { log::info!("Ehlo from {}", req.id()); + + let c_master = Arc::clone(&master); + let c_bcast = bcast.clone(); + let c_id = req.id().to_string(); + tokio::spawn(async move { + if !c_master.lock().await.mkr().is_registered(&c_id) { + log::info!("Not registered"); + c_master.lock().await.to_drop.insert(minion_addr); + _ = c_bcast.send( + c_master.lock().await.msg_not_registered(req.id().to_string()).sendable().unwrap(), + ); + } else { + log::info!("Registered"); + } + }); } _ => { log::error!("Minion sends unknown request type"); @@ -138,60 +174,84 @@ impl SysMaster { }); } - pub async fn do_outgoing( - master: Arc>, tx: tokio::sync::mpsc::Sender<(Vec, usize)>, - ) -> Result<(), SysinspectError> { + pub async fn do_outgoing(master: Arc>, tx: mpsc::Sender<(Vec, String)>) -> Result<(), SysinspectError> { log::trace!("Init outgoing channel"); let listener = master.lock().await.listener().await?; tokio::spawn(async move { let bcast = master.lock().await.broadcast(); - let mut client_id_counter: usize = 0; + loop { - if let Ok((socket, _)) = listener.accept().await { - client_id_counter += 1; - let current_client_id = client_id_counter; - let mut rx = bcast.subscribe(); - let client_tx = tx.clone(); - - let (reader, writer) = socket.into_split(); - - // Task to send messages to the client - tokio::spawn(async move { - let mut writer = writer; - log::info!("Minion {} connected. Ready to send messages.", current_client_id); - loop { - if let Ok(msg) = rx.recv().await { - log::info!("Sending message to client {}: {:?}", current_client_id, msg); - if writer.write_all(&(msg.len() as u32).to_be_bytes()).await.is_err() - || writer.write_all(&msg).await.is_err() - || writer.flush().await.is_err() - { - break; + tokio::select! { + // Accept a new connection + Ok((socket, _)) = listener.accept() => { + let mut bcast_sub = bcast.subscribe(); + let client_tx = tx.clone(); + let local_addr = socket.local_addr().unwrap(); + let (reader, writer) = socket.into_split(); + let c_master = Arc::clone(&master); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + + // Task to send messages to the client + tokio::spawn(async move { + let mut writer = writer; + log::info!("Minion {} connected. Ready to send messages.", local_addr.to_string()); + + loop { + if let Ok(msg) = bcast_sub.recv().await { + log::trace!("Sending message to minion at {} length of {}", local_addr.to_string(), msg.len()); + if writer.write_all(&(msg.len() as u32).to_be_bytes()).await.is_err() + || writer.write_all(&msg).await.is_err() + || writer.flush().await.is_err() + { + if let Err(err) = cancel_tx.send(true) { + log::error!("Sending cancel notification: {err}"); + } + break; + } + + if c_master.lock().await.to_drop.contains(&local_addr.to_string()) { + c_master.lock().await.to_drop.remove(&local_addr.to_string()); + log::info!("Dropping minion: {}", &local_addr.to_string()); + if let Err(err) = writer.shutdown().await { + log::error!("Error shutting down outgoing: {err}"); + } + if let Err(err) = cancel_tx.send(true) { + log::error!("Sending cancel notification: {err}"); + } + + return; + } } } - } - }); + }); - // Task to read messages from the client - tokio::spawn(async move { - let mut reader = TokioBufReader::new(reader); - loop { - let mut len_buf = [0u8; 4]; - if reader.read_exact(&mut len_buf).await.is_err() { - return; - } + // Task to read messages from the client + tokio::spawn(async move { + let mut reader = TokioBufReader::new(reader); + loop { + if *cancel_rx.borrow() { + log::info!("Process terminated"); + return; + } - let msg_len = u32::from_be_bytes(len_buf) as usize; - let mut msg = vec![0u8; msg_len]; - if reader.read_exact(&mut msg).await.is_err() { - return; - } + let mut len_buf = [0u8; 4]; + if reader.read_exact(&mut len_buf).await.is_err() { + return; + } + + let msg_len = u32::from_be_bytes(len_buf) as usize; + let mut msg = vec![0u8; msg_len]; + if reader.read_exact(&mut msg).await.is_err() { + return; + } + + if client_tx.send((msg, local_addr.to_string())).await.is_err() { + break; + } - if client_tx.send((msg, current_client_id)).await.is_err() { - break; } - } - }); + }); + } } } }); @@ -201,13 +261,13 @@ impl SysMaster { } pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { - let master = Arc::new(Mutex::new(SysMaster::new(cfg))); + let master = Arc::new(Mutex::new(SysMaster::new(cfg)?)); { let mut m = master.lock().await; m.init().await?; } - let (client_tx, client_rx) = mpsc::channel::<(Vec, usize)>(100); + let (client_tx, client_rx) = mpsc::channel::<(Vec, String)>(100); // Task to read from the FIFO and broadcast messages to clients SysMaster::do_fifo(Arc::clone(&master)).await; From 3165d64ebb8cd0b55838f8c0bc6f93d99b07d79a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 18:11:48 +0100 Subject: [PATCH 049/148] Change type of db search --- sysmaster/src/registry/mkb.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sysmaster/src/registry/mkb.rs b/sysmaster/src/registry/mkb.rs index 70d34e35..b903bddc 100644 --- a/sysmaster/src/registry/mkb.rs +++ b/sysmaster/src/registry/mkb.rs @@ -36,7 +36,7 @@ impl MinionKeyRegistry { } /// Returns a method if a minion Id is known to the key registry. - pub fn is_registered(&self, mid: String) -> bool { - self.keys.contains_key(&mid) + pub fn is_registered(&self, mid: &str) -> bool { + self.keys.contains_key(mid) } } From 440432a41928f52f4b14138b1b0c4b60ee8281a1 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 18:12:10 +0100 Subject: [PATCH 050/148] Move proto code to its own module --- sysminion/src/main.rs | 1 + sysminion/src/minion.rs | 58 +++++++++++++++++++++++++---------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 2fb81ac0..1f6624d3 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -1,6 +1,7 @@ mod clidef; mod config; mod minion; +mod proto; mod traits; use clidef::cli; diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index c515c90a..a6af1659 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,10 +1,9 @@ use crate::{ - config::{self, MinionConfig}, - traits, + config::{self}, + proto, }; use libsysinspect::{ - proto::{rqtypes::RequestType, MinionMessage, ProtoConversion}, - util::{self, dataconv}, + util::{self}, SysinspectError, }; use std::{path::PathBuf, sync::Arc}; @@ -16,19 +15,6 @@ use tokio::{ sync::mpsc, }; -/// Send ehlo -async fn send_ehlo(stream: Arc>, cfg: MinionConfig) -> Result<(), SysinspectError> { - let r = MinionMessage::new( - dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), - RequestType::Ehlo, - "".to_string(), - ); - - log::info!("Ehlo on {}", cfg.master()); - request(stream, r.sendable()?).await; - Ok(()) -} - /// Talk-back to the master pub async fn request(stream: Arc>, msg: Vec) { let mut stm = stream.lock().await; @@ -63,7 +49,7 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { let (_w_chan, mut r_chan) = mpsc::channel(100); // ehlo - send_ehlo(wstm.clone(), cfg).await?; + proto::msg::send_ehlo(wstm.clone(), cfg).await?; // Data exchange let wtsm_c = wstm.clone(); @@ -72,7 +58,7 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { loop { let mut buff = [0u8; 4]; if let Err(e) = input.read_exact(&mut buff).await { - log::error!("Unknown message length from the master: {}", e); + log::trace!("Unknown message length from the master: {}", e); break; } let msg_len = u32::from_be_bytes(buff) as usize; @@ -83,11 +69,37 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { break; } - log::info!("Received: {}", String::from_utf8_lossy(&msg)); + match proto::msg::get_message(msg) { + Ok(msg) => { + log::debug!("Received: {:?}", msg); + match msg.req_type() { + libsysinspect::proto::rqtypes::RequestType::Add => { + log::debug!("Master asks to register"); + } + libsysinspect::proto::rqtypes::RequestType::Remove => { + log::debug!("Master asks to unregister"); + } + libsysinspect::proto::rqtypes::RequestType::Command => { + log::debug!("Master sends a command"); + } + libsysinspect::proto::rqtypes::RequestType::Traits => { + log::debug!("Master requests traits"); + } + libsysinspect::proto::rqtypes::RequestType::AgentUnknown => { + log::info!("{}", msg.payload()); // Unknowns are NOT encrypted. + std::process::exit(1); + } + _ => { + log::error!("Unknown request type"); + } + } - // Send a response back to the master after receiving each message - let response = format!("Back: '{}'", String::from_utf8_lossy(&msg)).as_bytes().to_vec(); - request(wtsm_c.clone(), response).await; + //request(wtsm_c.clone(), response).await; + } + Err(err) => { + log::error!("{err}"); + } + } } }); From 6c8677486824843e0d5117f00f0facc1d06f1e77 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 20:19:04 +0100 Subject: [PATCH 051/148] Handle RSA dynamic errors --- libsysinspect/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index fa234e00..0a69e5c0 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -32,6 +32,7 @@ pub enum SysinspectError { SerdeYaml(serde_yaml::Error), SerdeJson(serde_json::Error), FFINullError(NulError), + DynError(Box), } impl Error for SysinspectError { @@ -59,6 +60,7 @@ impl Display for SysinspectError { SysinspectError::MasterGeneralError(err) => format!("(Master) {err}"), SysinspectError::MinionGeneralError(err) => format!("(Minion) {err}"), SysinspectError::ProtoError(err) => format!("(Protocol) {err}"), + SysinspectError::DynError(err) => format!("(General) {err}"), }; write!(f, "{msg}")?; @@ -93,3 +95,10 @@ impl From for SysinspectError { SysinspectError::FFINullError(err) } } + +// Implement From> for SysinspectError +impl From> for SysinspectError { + fn from(err: Box) -> SysinspectError { + SysinspectError::DynError(err) + } +} From 42a182979e54bb15b19823c60d957977880c9175 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 20:19:23 +0100 Subject: [PATCH 052/148] Implement RSA keys autogenerate --- sysmaster/src/registry/mkb.rs | 32 +++++++++++++++++++++++++++++--- sysmaster/src/registry/mod.rs | 2 ++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/sysmaster/src/registry/mkb.rs b/sysmaster/src/registry/mkb.rs index b903bddc..bd210aa3 100644 --- a/sysmaster/src/registry/mkb.rs +++ b/sysmaster/src/registry/mkb.rs @@ -1,5 +1,5 @@ -use super::{CFG_DEFAULT_ROOT, CFG_MINION_KEYS}; -use libsysinspect::SysinspectError; +use super::{CFG_DEFAULT_ROOT, CFG_MASTER_KEY_PRI, CFG_MASTER_KEY_PUB, CFG_MINION_KEYS}; +use libsysinspect::{rsa, SysinspectError}; use std::{collections::HashMap, fs, path::PathBuf}; /// Registered minion base. @@ -21,6 +21,32 @@ impl MinionKeyRegistry { Ok(reg) } + /// Generate keys, if none + fn gen_keys(&self) -> Result<(), SysinspectError> { + let prk_pth = self.root.parent().unwrap().join(CFG_MASTER_KEY_PRI); + let pbk_pth = self.root.parent().unwrap().join(CFG_MASTER_KEY_PUB); + + if prk_pth.exists() || pbk_pth.exists() { + return Ok(()); + } + + log::debug!("Generating RSA keys..."); + + let (prk, pbk) = rsa::keys::keygen(rsa::keys::DEFAULT_KEY_SIZE)?; + let (prk_pem, pbk_pem) = rsa::keys::to_pem(Some(&prk), Some(&pbk))?; + + if prk_pem.is_none() || pbk_pem.is_none() { + return Err(SysinspectError::MasterGeneralError(format!("Unable to generate RSA keys"))); + } + + fs::write(prk_pth, prk_pem.unwrap().as_bytes())?; + fs::write(pbk_pth, pbk_pem.unwrap().as_bytes())?; + + log::debug!("RSA keys saved to the disk"); + + Ok(()) + } + /// Sets up the registry fn setup(&mut self) -> Result<(), SysinspectError> { if !self.root.exists() { @@ -32,7 +58,7 @@ impl MinionKeyRegistry { } } - Ok(()) + self.gen_keys() } /// Returns a method if a minion Id is known to the key registry. diff --git a/sysmaster/src/registry/mod.rs b/sysmaster/src/registry/mod.rs index 497412e2..55f53d15 100644 --- a/sysmaster/src/registry/mod.rs +++ b/sysmaster/src/registry/mod.rs @@ -13,6 +13,8 @@ use sled::Db; pub static CFG_DEFAULT_ROOT: &str = "/etc/sysinspect"; pub static CFG_DB: &str = "registry"; pub static CFG_MINION_KEYS: &str = "minion-keys"; +pub static CFG_MASTER_KEY_PUB: &str = "master.rsa.pub"; +pub static CFG_MASTER_KEY_PRI: &str = "master.rsa"; pub struct MinionRegistry { conn: Db, From f75f93b4470ba30f8d2ca46eade007f39854d41f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 20:19:33 +0100 Subject: [PATCH 053/148] Add CLI option to register a minion --- sysminion/src/clidef.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sysminion/src/clidef.rs b/sysminion/src/clidef.rs index 0602733f..e050474f 100644 --- a/sysminion/src/clidef.rs +++ b/sysminion/src/clidef.rs @@ -23,10 +23,10 @@ pub fn cli(version: &'static str, appname: &'static str) -> Command { .help("Alternative path to the config") ) .arg( - Arg::new("master") + Arg::new("register") .short('r') - .long("master") - .help("Register to the master by Id") // XXX: This must be a key fingerprint in a future + .long("register") + .help("Register to the master by its fingerprint") // XXX: This must be a key fingerprint in a future ) .arg( Arg::new("start") From 51d7b6677dbcac972aa63c61b598b94a244cb9ec Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 20:24:02 +0100 Subject: [PATCH 054/148] Update log records --- sysmaster/src/master.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 78707f5a..c2c7d0a5 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -107,14 +107,14 @@ impl SysMaster { log::info!("Response"); } RequestType::Ehlo => { - log::info!("Ehlo from {}", req.id()); + log::info!("EHLO from {}", req.id()); let c_master = Arc::clone(&master); let c_bcast = bcast.clone(); let c_id = req.id().to_string(); tokio::spawn(async move { if !c_master.lock().await.mkr().is_registered(&c_id) { - log::info!("Not registered"); + log::info!("Minion at {minion_addr} ({}) is not registered", req.id()); c_master.lock().await.to_drop.insert(minion_addr); _ = c_bcast.send( c_master.lock().await.msg_not_registered(req.id().to_string()).sendable().unwrap(), From 7cc1e69108e5e8b38be7526cf832c36cf9b5162e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 22:11:20 +0100 Subject: [PATCH 055/148] Add keymanager for the minion --- sysminion/src/rsa.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 sysminion/src/rsa.rs diff --git a/sysminion/src/rsa.rs b/sysminion/src/rsa.rs new file mode 100644 index 00000000..f7f762b8 --- /dev/null +++ b/sysminion/src/rsa.rs @@ -0,0 +1,81 @@ +/* +RSA keys manager + */ + +use std::{fs, path::PathBuf}; + +use libsysinspect::SysinspectError; +use rsa::{RsaPrivateKey, RsaPublicKey}; + +static CFG_DEFAULT_MINION_ROOT: &str = "/etc/sysinspect"; +static CFG_MINION_RSA_PUB: &str = "minion.rsa.pub"; +static CFG_MINION_RSA_PRV: &str = "minion.rsa"; + +#[derive(Debug, Default, Clone)] +pub struct MinionRSAKeyManager { + root: PathBuf, + + // RSA + mn_prk: Option, + mn_pbk: Option, + mn_pbk_pem: String, +} + +impl MinionRSAKeyManager { + /// Initiate Minion's RSA key manager. Parameter `root` is + /// optional, if configuration contains alternative Minion root. + pub fn new(root: Option) -> Result { + let mut keyman = MinionRSAKeyManager { + root: PathBuf::from(root.unwrap_or(CFG_DEFAULT_MINION_ROOT.to_string())), + ..Default::default() + }; + + keyman.setup()?; + Ok(keyman) + } + + /// Initialise RSA keys, if none + fn init_keys(&mut self) -> Result<(), SysinspectError> { + let prk_pth = self.root.join(CFG_MINION_RSA_PRV); + let pbk_pth = self.root.join(CFG_MINION_RSA_PUB); + + // Exists already? + if prk_pth.exists() || pbk_pth.exists() { + let prk_pem = fs::read_to_string(prk_pth)?; + let pbk_pem = fs::read_to_string(pbk_pth)?; + (self.mn_prk, self.mn_pbk) = libsysinspect::rsa::keys::from_pem(Some(&prk_pem), Some(&pbk_pem))?; + self.mn_pbk_pem = pbk_pem; + + return Ok(()); + } + + // Create RSA keypair + log::info!("Creating RSA keypair..."); + + let (prk, pbk) = libsysinspect::rsa::keys::keygen(libsysinspect::rsa::keys::DEFAULT_KEY_SIZE)?; + let (prk_pem, pbk_pem) = libsysinspect::rsa::keys::to_pem(Some(&prk), Some(&pbk))?; + + if prk_pem.is_none() || pbk_pem.is_none() { + return Err(SysinspectError::MinionGeneralError("Error generating new RSA keys".to_string())); + } + + self.mn_pbk_pem = pbk_pem.to_owned().unwrap(); + fs::write(prk_pth, prk_pem.unwrap())?; + fs::write(pbk_pth, pbk_pem.unwrap())?; + + log::info!("RSA keypair created"); + + Ok(()) + } + + /// Setup the RSA key manager + fn setup(&mut self) -> Result<(), SysinspectError> { + self.init_keys()?; + Ok(()) + } + + /// Get RSA PEM pubkey + pub fn get_pubkey_pem(&self) -> String { + self.mn_pbk_pem.to_owned() + } +} From d61640b8acb5b6f194ba86835df04f4932ad813f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 6 Nov 2024 22:11:34 +0100 Subject: [PATCH 056/148] Update dependencies --- Cargo.lock | 2 ++ sysmaster/Cargo.toml | 1 + sysminion/Cargo.toml | 1 + 3 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 1ea4706f..70d102bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2015,6 +2015,7 @@ dependencies = [ "libsysinspect", "log", "rand", + "rsa", "rustls", "rustls-pemfile", "serde", @@ -2037,6 +2038,7 @@ dependencies = [ "log", "once_cell", "rand", + "rsa", "rustls", "rustls-pemfile", "serde", diff --git a/sysmaster/Cargo.toml b/sysmaster/Cargo.toml index 67793754..6db76d6e 100644 --- a/sysmaster/Cargo.toml +++ b/sysmaster/Cargo.toml @@ -30,3 +30,4 @@ tokio-rustls = "0.26.0" libsysinspect = { path = "../libsysinspect" } log = "0.4.22" sled = "0.34.7" +rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } diff --git a/sysminion/Cargo.toml b/sysminion/Cargo.toml index 8bbda79c..7b5c25c0 100644 --- a/sysminion/Cargo.toml +++ b/sysminion/Cargo.toml @@ -29,3 +29,4 @@ serde_json = "1.0.132" indexmap = "2.6.0" once_cell = "1.20.2" sysinfo = { version = "0.32.0", features = ["linux-tmpfs"] } +rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } From bda66c078e9745f46fede4bafcbc90e6de9c0874 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 01:37:32 +0100 Subject: [PATCH 057/148] Add reconnect request type --- libsysinspect/src/proto/rqtypes.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libsysinspect/src/proto/rqtypes.rs b/libsysinspect/src/proto/rqtypes.rs index 82e4abb4..78e878b6 100644 --- a/libsysinspect/src/proto/rqtypes.rs +++ b/libsysinspect/src/proto/rqtypes.rs @@ -26,6 +26,10 @@ pub enum RequestType { #[serde(rename = "ehlo")] Ehlo, + /// Retry connect (e.g. after the registration) + #[serde(rename = "retry")] + Reconnect, + /// Unknown agent #[serde(rename = "undef")] AgentUnknown, From 46f8dee1ad4ea4ef1190d0844624583f772f69e5 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 01:37:47 +0100 Subject: [PATCH 058/148] Add keypair registration --- sysmaster/src/master.rs | 51 +++++++++++++++----- sysmaster/src/registry/mkb.rs | 88 +++++++++++++++++++++++++++++++---- 2 files changed, 118 insertions(+), 21 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index c2c7d0a5..3411859b 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -1,9 +1,8 @@ -use crate::{config::MasterConfig, registry::mkb::MinionKeyRegistry}; +use crate::{config::MasterConfig, registry::mkb::MinionsKeyRegistry}; use libsysinspect::{ proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; -use rustls::crypto::hash::Hash; use std::{collections::HashSet, path::Path, sync::Arc}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as TokioBufReader}; use tokio::net::TcpListener; @@ -16,7 +15,7 @@ use tokio::{fs::OpenOptions, sync::Mutex}; pub struct SysMaster { cfg: MasterConfig, broadcast: broadcast::Sender>, - mkr: MinionKeyRegistry, + mkr: MinionsKeyRegistry, to_drop: HashSet, } @@ -24,7 +23,7 @@ impl SysMaster { pub fn new(cfg: MasterConfig) -> Result { let (tx, _) = broadcast::channel::>(100); - let mkr = MinionKeyRegistry::new()?; + let mkr = MinionsKeyRegistry::new()?; Ok(SysMaster { cfg, broadcast: tx, mkr, to_drop: HashSet::default() }) } @@ -73,17 +72,28 @@ impl SysMaster { } /// Get Minion key registry - fn mkr(&self) -> &MinionKeyRegistry { - &self.mkr + fn mkr(&mut self) -> &mut MinionsKeyRegistry { + &mut self.mkr } /// Bounce message - fn msg_not_registered(&self, mid: String) -> MasterMessage { - let mut m = MasterMessage::new(RequestType::AgentUnknown, "Minion is not registered".to_string()); + fn msg_not_registered(&mut self, mid: String) -> MasterMessage { + let mut m = MasterMessage::new(RequestType::AgentUnknown, self.mkr().get_master_key_pem().clone().unwrap().to_string()); let mut tgt = MinionTarget::new(); tgt.add_minion_id(mid); m.add_target(tgt); - m.set_retcode(ProtoErrorCode::NotRegistered); + m.set_retcode(ProtoErrorCode::Success); + + m + } + + /// Accept registration + fn msg_registered(&self, mid: String, msg: &str) -> MasterMessage { + let mut m = MasterMessage::new(RequestType::Reconnect, msg.to_string()); // XXX: Should it be already encrypted? + let mut tgt = MinionTarget::new(); + tgt.add_minion_id(mid); + m.add_target(tgt); + m.set_retcode(ProtoErrorCode::Success); m } @@ -101,7 +111,26 @@ impl SysMaster { if let Some(req) = master.lock().await.to_request(&msg) { match req.req_type() { RequestType::Add => { - log::info!("Add"); + let c_master = Arc::clone(&master); + let c_bcast = bcast.clone(); + let c_mid = req.id().to_string(); + tokio::spawn(async move { + log::info!("Minion \"{}\" requested registration", minion_addr); + let mut guard = c_master.lock().await; + let resp_msg: &str; + if !guard.mkr().is_registered(&c_mid) { + if let Err(err) = guard.mkr().add_mn_key(&c_mid, &minion_addr, req.payload()) { + log::error!("Unable to add minion RSA key: {err}"); + } + guard.to_drop.insert(minion_addr.to_owned()); + resp_msg = "Minion registration has been accepted"; + log::info!("Registered a minion at {minion_addr} ({})", c_mid); + } else { + resp_msg = "Minion already registered"; + log::warn!("Minion {minion_addr} ({}) is already registered", c_mid); + } + _ = c_bcast.send(guard.msg_registered(req.id().to_string(), resp_msg).sendable().unwrap()); + }); } RequestType::Response => { log::info!("Response"); @@ -120,7 +149,7 @@ impl SysMaster { c_master.lock().await.msg_not_registered(req.id().to_string()).sendable().unwrap(), ); } else { - log::info!("Registered"); + log::info!("{} connected successfully", c_id); } }); } diff --git a/sysmaster/src/registry/mkb.rs b/sysmaster/src/registry/mkb.rs index bd210aa3..72709387 100644 --- a/sysmaster/src/registry/mkb.rs +++ b/sysmaster/src/registry/mkb.rs @@ -1,4 +1,5 @@ use super::{CFG_DEFAULT_ROOT, CFG_MASTER_KEY_PRI, CFG_MASTER_KEY_PUB, CFG_MINION_KEYS}; +use ::rsa::{RsaPrivateKey, RsaPublicKey}; use libsysinspect::{rsa, SysinspectError}; use std::{collections::HashMap, fs, path::PathBuf}; @@ -7,26 +8,39 @@ use std::{collections::HashMap, fs, path::PathBuf}; /// where collected all public keys from all minions. #[derive(Debug, Default, Clone)] -pub struct MinionKeyRegistry { +pub struct MinionsKeyRegistry { root: PathBuf, - keys: HashMap, + keys: HashMap>, + + // Master RSA + ms_prk: Option, + ms_pbk: Option, + ms_pbk_pem: Option, } -impl MinionKeyRegistry { - pub fn new() -> Result { +impl MinionsKeyRegistry { + pub fn new() -> Result { let mut reg = - MinionKeyRegistry { root: PathBuf::from(CFG_DEFAULT_ROOT).join(CFG_MINION_KEYS), ..MinionKeyRegistry::default() }; + MinionsKeyRegistry { root: PathBuf::from(CFG_DEFAULT_ROOT).join(CFG_MINION_KEYS), ..MinionsKeyRegistry::default() }; reg.setup()?; Ok(reg) } /// Generate keys, if none - fn gen_keys(&self) -> Result<(), SysinspectError> { + fn init_keys(&mut self) -> Result<(), SysinspectError> { let prk_pth = self.root.parent().unwrap().join(CFG_MASTER_KEY_PRI); let pbk_pth = self.root.parent().unwrap().join(CFG_MASTER_KEY_PUB); if prk_pth.exists() || pbk_pth.exists() { + let prk_pem = fs::read_to_string(prk_pth)?; + self.ms_pbk_pem = Some(fs::read_to_string(pbk_pth)?); + (self.ms_prk, self.ms_pbk) = rsa::keys::from_pem(Some(&prk_pem), self.ms_pbk_pem.as_deref())?; + + if self.ms_pbk.is_none() || self.ms_pbk.is_none() { + return Err(SysinspectError::MasterGeneralError(format!("Unable to initialise RSA keys"))); + } + return Ok(()); } @@ -40,7 +54,9 @@ impl MinionKeyRegistry { } fs::write(prk_pth, prk_pem.unwrap().as_bytes())?; - fs::write(pbk_pth, pbk_pem.unwrap().as_bytes())?; + fs::write(pbk_pth, pbk_pem.clone().unwrap().as_bytes())?; + + self.ms_pbk_pem = pbk_pem; log::debug!("RSA keys saved to the disk"); @@ -53,16 +69,68 @@ impl MinionKeyRegistry { fs::create_dir_all(&self.root)?; } else { for e in fs::read_dir(&self.root)?.flatten() { - self.keys - .insert(e.file_name().to_str().and_then(|e| e.split('.').next()).unwrap_or_default().to_string(), e.path()); + self.keys.insert(e.file_name().to_str().and_then(|e| e.split('.').next()).unwrap_or_default().to_string(), None); } } - self.gen_keys() + self.init_keys() } /// Returns a method if a minion Id is known to the key registry. pub fn is_registered(&self, mid: &str) -> bool { self.keys.contains_key(mid) } + + /// Get a fingerprint of a master key + pub fn get_master_key_pem(&self) -> &Option { + &self.ms_pbk_pem + } + + /// Add minion key + pub fn add_mn_key(&mut self, mid: &str, addr: &str, pbk_pem: &str) -> Result<(), SysinspectError> { + let k_pth = self.root.join(format!("{}.rsa.pub", mid)); + log::debug!("Adding minion key for {mid} at {addr} as {}", k_pth.as_os_str().to_str().unwrap_or_default()); + fs::write(k_pth, pbk_pem)?; + + let (_, pbk) = rsa::keys::from_pem(None, Some(pbk_pem))?; + if let Some(pbk) = pbk { + self.keys.insert(mid.to_string(), Some(pbk)); + } + Ok(()) + } + + /// Lazy-load minion key. By start all keys are only containing minion Ids. + /// If a key is requested, it is loaded from the disk on demand. + fn get_mn_key(&mut self, mid: &str) -> Option { + log::debug!("Loading RSA key for {mid}"); + + if let Some(pbk) = self.keys.get(mid).and_then(|s| s.clone()) { + return Some(pbk); + } + + let k_pth = self.root.join(format!("{}.rsa.pub", mid)); + if !k_pth.exists() { + log::error!("Minion {mid} requests RSA key, but the key is not found!"); + return None; + } + + match fs::read_to_string(k_pth) { + Ok(pbk_pem) => { + if let Ok((_, pbk)) = rsa::keys::from_pem(None, Some(&pbk_pem)) { + if let Some(pbk) = pbk { + self.keys.insert(mid.to_string(), Some(pbk.to_owned())); + return Some(pbk); + } + } + } + Err(err) => log::error!("Unable to read minion RSA key: {err}"), + } + None + } + + pub fn remove_mn_key(&self) {} + + pub fn encrypt_with_mn_key(&self) {} + + pub fn encrypt_with_mst_key(&self) {} } From 940717369b0a85a5b0b6ec187d5193f43ce30d92 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 01:38:17 +0100 Subject: [PATCH 059/148] Prevent "start" be together with "register" --- sysminion/src/clidef.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sysminion/src/clidef.rs b/sysminion/src/clidef.rs index e050474f..8b9581d7 100644 --- a/sysminion/src/clidef.rs +++ b/sysminion/src/clidef.rs @@ -26,6 +26,7 @@ pub fn cli(version: &'static str, appname: &'static str) -> Command { Arg::new("register") .short('r') .long("register") + .conflicts_with("start") .help("Register to the master by its fingerprint") // XXX: This must be a key fingerprint in a future ) .arg( From ffa5b383656b578ef2f106701448ef59f7c935c3 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 01:38:31 +0100 Subject: [PATCH 060/148] Use registration CLI --- sysminion/src/main.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 1f6624d3..620dc5a3 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -2,6 +2,7 @@ mod clidef; mod config; mod minion; mod proto; +mod rsa; mod traits; use clidef::cli; @@ -46,9 +47,10 @@ async fn main() -> std::io::Result<()> { } // Start - if *params.get_one::("start").unwrap_or(&false) { + let fp = params.get_one::("register").cloned(); + if *params.get_one::("start").unwrap_or(&false) || fp.is_some() { let cfp = params.get_one::("config"); - if let Err(err) = minion::minion(PathBuf::from(cfp.map_or("", |v| v))).await { + if let Err(err) = minion::minion(PathBuf::from(cfp.map_or("", |v| v)), fp).await { log::error!("Unable to start minion: {}", err); return Ok(()); } From 5fb23afb2c43f24073cbc365961650c579f2a57f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 01:38:48 +0100 Subject: [PATCH 061/148] Implement registration on minion side --- sysminion/src/minion.rs | 34 +++++++++++++++++++++++++++------- sysminion/src/proto.rs | 13 ++++++++++++- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index a6af1659..0423282f 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -3,6 +3,7 @@ use crate::{ proto, }; use libsysinspect::{ + rsa, util::{self}, SysinspectError, }; @@ -37,20 +38,19 @@ pub async fn request(stream: Arc>, msg: Vec) { } /// Minion routine -pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { +pub async fn minion(mut cfp: PathBuf, fingerprint: Option) -> Result<(), SysinspectError> { + let fingerprint = fingerprint.unwrap_or_default(); if !cfp.exists() { cfp = util::cfg::select_config()?; } let cfg = config::MinionConfig::new(cfp)?; //let st = traits::get_traits(); + let mkeys = crate::rsa::MinionRSAKeyManager::new(None)?; // XXX: Get optional root from the configuration let (rstm, wstm) = TcpStream::connect(cfg.master()).await?.into_split(); let wstm: Arc> = Arc::new(Mutex::new(wstm)); let (_w_chan, mut r_chan) = mpsc::channel(100); - // ehlo - proto::msg::send_ehlo(wstm.clone(), cfg).await?; - // Data exchange let wtsm_c = wstm.clone(); tokio::spawn(async move { @@ -69,13 +69,20 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { break; } - match proto::msg::get_message(msg) { + match proto::msg::payload_to_msg(msg) { Ok(msg) => { log::debug!("Received: {:?}", msg); match msg.req_type() { libsysinspect::proto::rqtypes::RequestType::Add => { - log::debug!("Master asks to register"); + log::debug!("Master accepts registration"); + } + + libsysinspect::proto::rqtypes::RequestType::Reconnect => { + log::debug!("Master requires reconnection"); + log::info!("{}", msg.payload()); + std::process::exit(0); } + libsysinspect::proto::rqtypes::RequestType::Remove => { log::debug!("Master asks to unregister"); } @@ -86,7 +93,12 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { log::debug!("Master requests traits"); } libsysinspect::proto::rqtypes::RequestType::AgentUnknown => { - log::info!("{}", msg.payload()); // Unknowns are NOT encrypted. + let pbk_pem = msg.payload(); // Expected PEM RSA pub key + let (_, pbk) = rsa::keys::from_pem(None, Some(pbk_pem)).unwrap(); + let fpt = rsa::keys::get_fingerprint(&pbk.unwrap()).unwrap(); + + log::error!("Minion is not registered"); + log::info!("Master fingerprint: {}", fpt); std::process::exit(1); } _ => { @@ -111,6 +123,14 @@ pub async fn minion(mut cfp: PathBuf) -> Result<(), SysinspectError> { } }); + // Messages + if !fingerprint.is_empty() { + proto::msg::send_registration(wstm.clone(), cfg, mkeys.get_pubkey_pem()).await?; + } else { + // ehlo + proto::msg::send_ehlo(wstm.clone(), cfg).await?; + } + // Keep the client alive until Ctrl+C is pressed tokio::signal::ctrl_c().await.expect("Failed to listen for ctrl_c"); log::info!("Shutting down client."); diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index 5a5024a2..90e26597 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -24,8 +24,19 @@ pub mod msg { Ok(()) } + pub async fn send_registration( + stream: Arc>, cfg: MinionConfig, pbk_pem: String, + ) -> Result<(), SysinspectError> { + let r = + MinionMessage::new(dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); + + log::info!("Registration request to {}", cfg.master()); + request(stream, r.sendable()?).await; + Ok(()) + } + /// Get message - pub fn get_message(data: Vec) -> Result { + pub fn payload_to_msg(data: Vec) -> Result { let data = match String::from_utf8(data) { Ok(data) => data, Err(err) => return Err(SysinspectError::ProtoError(format!("unable to parse master message: {err}"))), From 231d10866549244d64b4ce28dc0d3f413305a13e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 01:40:34 +0100 Subject: [PATCH 062/148] Remove formatter in favour to just a string --- sysmaster/src/registry/mkb.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sysmaster/src/registry/mkb.rs b/sysmaster/src/registry/mkb.rs index 72709387..ba997a98 100644 --- a/sysmaster/src/registry/mkb.rs +++ b/sysmaster/src/registry/mkb.rs @@ -38,7 +38,7 @@ impl MinionsKeyRegistry { (self.ms_prk, self.ms_pbk) = rsa::keys::from_pem(Some(&prk_pem), self.ms_pbk_pem.as_deref())?; if self.ms_pbk.is_none() || self.ms_pbk.is_none() { - return Err(SysinspectError::MasterGeneralError(format!("Unable to initialise RSA keys"))); + return Err(SysinspectError::MasterGeneralError("Unable to initialise RSA keys".to_string())); } return Ok(()); @@ -50,7 +50,7 @@ impl MinionsKeyRegistry { let (prk_pem, pbk_pem) = rsa::keys::to_pem(Some(&prk), Some(&pbk))?; if prk_pem.is_none() || pbk_pem.is_none() { - return Err(SysinspectError::MasterGeneralError(format!("Unable to generate RSA keys"))); + return Err(SysinspectError::MasterGeneralError("Unable to generate RSA keys".to_string())); } fs::write(prk_pth, prk_pem.unwrap().as_bytes())?; From 6f0ae75479bee5e17591c1d14b7dbfd7f9b9ce7a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 01:42:19 +0100 Subject: [PATCH 063/148] Flatten if clause --- sysmaster/src/registry/mkb.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sysmaster/src/registry/mkb.rs b/sysmaster/src/registry/mkb.rs index ba997a98..d5d477e4 100644 --- a/sysmaster/src/registry/mkb.rs +++ b/sysmaster/src/registry/mkb.rs @@ -116,11 +116,9 @@ impl MinionsKeyRegistry { match fs::read_to_string(k_pth) { Ok(pbk_pem) => { - if let Ok((_, pbk)) = rsa::keys::from_pem(None, Some(&pbk_pem)) { - if let Some(pbk) = pbk { - self.keys.insert(mid.to_string(), Some(pbk.to_owned())); - return Some(pbk); - } + if let Ok((_, Some(pbk))) = rsa::keys::from_pem(None, Some(&pbk_pem)) { + self.keys.insert(mid.to_string(), Some(pbk.to_owned())); + return Some(pbk); } } Err(err) => log::error!("Unable to read minion RSA key: {err}"), From eb5ac006d3ceb917fdc985ac977f377a84eb8f48 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 22:31:01 +0100 Subject: [PATCH 064/148] Update dependencies --- Cargo.lock | 11 +++++++++++ sysmaster/Cargo.toml | 1 + sysminion/Cargo.toml | 1 + 3 files changed, 13 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 70d102bf..354995b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2024,6 +2024,7 @@ dependencies = [ "sled", "tokio", "tokio-rustls", + "uuid", ] [[package]] @@ -2046,6 +2047,7 @@ dependencies = [ "serde_yaml", "sysinfo 0.32.0", "tokio", + "uuid", ] [[package]] @@ -2190,6 +2192,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +dependencies = [ + "getrandom", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/sysmaster/Cargo.toml b/sysmaster/Cargo.toml index 6db76d6e..dd20555a 100644 --- a/sysmaster/Cargo.toml +++ b/sysmaster/Cargo.toml @@ -31,3 +31,4 @@ libsysinspect = { path = "../libsysinspect" } log = "0.4.22" sled = "0.34.7" rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } +uuid = { version = "1.11.0", features = ["v4"] } diff --git a/sysminion/Cargo.toml b/sysminion/Cargo.toml index 7b5c25c0..1bdd18c0 100644 --- a/sysminion/Cargo.toml +++ b/sysminion/Cargo.toml @@ -30,3 +30,4 @@ indexmap = "2.6.0" once_cell = "1.20.2" sysinfo = { version = "0.32.0", features = ["linux-tmpfs"] } rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } +uuid = { version = "1.11.0", features = ["v4"] } From 7d7082d73a01c9a15f4e6eb17c38e8877c015906 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 22:31:18 +0100 Subject: [PATCH 065/148] Implement minion session tracker --- sysmaster/src/registry/session.rs | 88 +++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 sysmaster/src/registry/session.rs diff --git a/sysmaster/src/registry/session.rs b/sysmaster/src/registry/session.rs new file mode 100644 index 00000000..b14a95fb --- /dev/null +++ b/sysmaster/src/registry/session.rs @@ -0,0 +1,88 @@ +/* +Session keeper. +Keeps connected minions and updates their uptime via heartbeat. +This prevents simultaenous connection of multiple minions on the same machine. + */ + +use std::{collections::HashMap, time::Instant}; + +#[derive(Debug, Clone)] +struct Session { + uptime: Instant, + last: Instant, + sid: String, +} + +impl Session { + pub fn new(sid: &str) -> Session { + Session { last: Instant::now(), uptime: Instant::now(), sid: sid.to_string() } + } + + pub fn age_sec(&self) -> u64 { + self.last.elapsed().as_secs() + } + + pub fn update(&mut self) { + self.last = Instant::now() + } + + pub fn uptime_sec(&self) -> u64 { + self.uptime.elapsed().as_secs() + } + + pub fn session_id(&self) -> String { + self.sid.to_string() + } +} + +#[derive(Debug, Default, Clone)] +pub struct SessionKeeper { + sessions: HashMap, + lifetime: u64, +} + +impl SessionKeeper { + pub fn new(lifetime: u64) -> SessionKeeper { + SessionKeeper { lifetime, ..Default::default() } + } + + /// Collect the garbage (outdated sessions) + fn gc(&mut self) { + for s in self.sessions.keys().into_iter().map(|s| s.to_string()).collect::>() { + self.alive(&s); + } + } + + /// Create a new session or update the existing + pub fn ping(&mut self, mid: &str, sid: &str) { + self.sessions.entry(mid.to_string()).or_insert_with(|| Session::new(sid)).update(); + self.gc(); + } + + /// Return uptime for a minion (seconds) + pub fn uptime(&self, mid: &str) -> Option { + self.sessions.get(mid).and_then(|s| Some(s.uptime_sec())) + } + + /// Returns true if a minion is alive. + pub fn alive(&mut self, mid: &str) -> bool { + if let Some(session) = self.sessions.get(mid) { + if session.age_sec() < self.lifetime { + return true; + } + + self.sessions.remove(mid); + } + false + } + + pub(crate) fn exists(&mut self, mid: &str) -> bool { + self.gc(); + self.sessions.contains_key(mid) + } + + /// Get session Id for the minion + pub(crate) fn get_id(&self, mid: &str) -> Option { + self.sessions.get(mid).and_then(|s| Some(s.session_id())) + } +} From df92cca1012fc999965ae4226a2ca6a8c996c6be Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 22:31:40 +0100 Subject: [PATCH 066/148] Add async dynamic error --- libsysinspect/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 0a69e5c0..5fda82fc 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -33,6 +33,7 @@ pub enum SysinspectError { SerdeJson(serde_json::Error), FFINullError(NulError), DynError(Box), + AsynDynError(Box), } impl Error for SysinspectError { @@ -61,6 +62,7 @@ impl Display for SysinspectError { SysinspectError::MinionGeneralError(err) => format!("(Minion) {err}"), SysinspectError::ProtoError(err) => format!("(Protocol) {err}"), SysinspectError::DynError(err) => format!("(General) {err}"), + SysinspectError::AsynDynError(err) => format!("(General part) {err}"), }; write!(f, "{msg}")?; @@ -102,3 +104,9 @@ impl From> for SysinspectError { SysinspectError::DynError(err) } } + +impl From> for SysinspectError { + fn from(err: Box) -> SysinspectError { + SysinspectError::AsynDynError(err) + } +} From c0d53acbac670968acdc8145b38ff5fce8848fa4 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 22:32:45 +0100 Subject: [PATCH 067/148] Implement heartbeat and unique connections per machine (not per net address) --- libsysinspect/src/proto/errcodes.rs | 3 + libsysinspect/src/proto/mod.rs | 5 +- libsysinspect/src/proto/rqtypes.rs | 8 +++ sysmaster/src/master.rs | 74 +++++++++++++++++++--- sysmaster/src/registry/mod.rs | 1 + sysminion/src/minion.rs | 95 ++++++++++++++++++----------- sysminion/src/proto.rs | 22 ++++++- 7 files changed, 161 insertions(+), 47 deletions(-) diff --git a/libsysinspect/src/proto/errcodes.rs b/libsysinspect/src/proto/errcodes.rs index ef218ed2..68f75a63 100644 --- a/libsysinspect/src/proto/errcodes.rs +++ b/libsysinspect/src/proto/errcodes.rs @@ -17,6 +17,9 @@ pub enum ProtoErrorCode { /// Minion is already registered AlreadyRegistered = 4, + /// Minion is already connected + AlreadyConnected = 5, + /// Unassigned, unknown Unknown, } diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 119d7c6a..82ea22cb 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -48,6 +48,7 @@ impl MasterMessage { 2 => ProtoErrorCode::GeneralFailure, 3 => ProtoErrorCode::NotRegistered, 4 => ProtoErrorCode::AlreadyRegistered, + 5 => ProtoErrorCode::AlreadyConnected, _ => ProtoErrorCode::Unknown, } } @@ -147,8 +148,8 @@ impl MinionTarget { } /// Add hostnames - pub fn add_hostname(&mut self, hostname: String) { - self.hostnames.push(hostname); + pub fn add_hostname(&mut self, hostname: &str) { + self.hostnames.push(hostname.to_string()); } } diff --git a/libsysinspect/src/proto/rqtypes.rs b/libsysinspect/src/proto/rqtypes.rs index 78e878b6..025edfb6 100644 --- a/libsysinspect/src/proto/rqtypes.rs +++ b/libsysinspect/src/proto/rqtypes.rs @@ -33,4 +33,12 @@ pub enum RequestType { /// Unknown agent #[serde(rename = "undef")] AgentUnknown, + + /// Ping + #[serde(rename = "pi")] + Ping, + + /// Pong + #[serde(rename = "po")] + Pong, } diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 3411859b..94399b48 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -1,15 +1,24 @@ -use crate::{config::MasterConfig, registry::mkb::MinionsKeyRegistry}; +use crate::{ + config::MasterConfig, + registry::{ + mkb::MinionsKeyRegistry, + session::{self, SessionKeeper}, + }, +}; use libsysinspect::{ proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; use std::{collections::HashSet, path::Path, sync::Arc}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as TokioBufReader}; use tokio::net::TcpListener; use tokio::select; use tokio::sync::{broadcast, mpsc}; use tokio::time::{sleep, Duration}; use tokio::{fs::OpenOptions, sync::Mutex}; +use tokio::{ + io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as TokioBufReader}, + time, +}; #[derive(Debug)] pub struct SysMaster { @@ -17,14 +26,14 @@ pub struct SysMaster { broadcast: broadcast::Sender>, mkr: MinionsKeyRegistry, to_drop: HashSet, + session: session::SessionKeeper, } impl SysMaster { pub fn new(cfg: MasterConfig) -> Result { let (tx, _) = broadcast::channel::>(100); - let mkr = MinionsKeyRegistry::new()?; - Ok(SysMaster { cfg, broadcast: tx, mkr, to_drop: HashSet::default() }) + Ok(SysMaster { cfg, broadcast: tx, mkr, to_drop: HashSet::default(), session: SessionKeeper::new(30) }) } /// Open FIFO socket for command-line communication @@ -76,6 +85,17 @@ impl SysMaster { &mut self.mkr } + /// Already connected + fn msg_already_connected(&mut self, mid: String, sid: String) -> MasterMessage { + let mut m = MasterMessage::new(RequestType::Command, sid); + let mut tgt = MinionTarget::new(); + tgt.add_minion_id(mid); + m.add_target(tgt); + m.set_retcode(ProtoErrorCode::AlreadyConnected); + + m + } + /// Bounce message fn msg_not_registered(&mut self, mid: String) -> MasterMessage { let mut m = MasterMessage::new(RequestType::AgentUnknown, self.mkr().get_master_key_pem().clone().unwrap().to_string()); @@ -141,18 +161,41 @@ impl SysMaster { let c_master = Arc::clone(&master); let c_bcast = bcast.clone(); let c_id = req.id().to_string(); + let c_payload = req.payload().to_string(); tokio::spawn(async move { - if !c_master.lock().await.mkr().is_registered(&c_id) { + let mut guard = c_master.lock().await; + if !guard.mkr().is_registered(&c_id) { log::info!("Minion at {minion_addr} ({}) is not registered", req.id()); - c_master.lock().await.to_drop.insert(minion_addr); + guard.to_drop.insert(minion_addr); + _ = c_bcast.send(guard.msg_not_registered(req.id().to_string()).sendable().unwrap()); + } else if guard.session.exists(&c_id) { + log::info!("Minion at {minion_addr} ({}) is already connected", req.id()); + guard.to_drop.insert(minion_addr); _ = c_bcast.send( - c_master.lock().await.msg_not_registered(req.id().to_string()).sendable().unwrap(), + guard.msg_already_connected(req.id().to_string(), c_payload).sendable().unwrap(), ); } else { log::info!("{} connected successfully", c_id); + guard.session.ping(&c_id, &c_payload); } }); } + + RequestType::Pong => { + let c_master = Arc::clone(&master); + let c_id = req.id().to_string(); + let c_payload = req.payload().to_string(); + tokio::spawn(async move { + let mut guard = c_master.lock().await; + guard.session.ping(&c_id, &c_payload); + let uptime = guard.session.uptime(req.id()).unwrap_or_default(); + log::debug!( + "Update last contacted for {} (alive for {:.2} min)", + req.id().to_string(), + uptime as f64 / 60.0 + ); + }); + } _ => { log::error!("Minion sends unknown request type"); } @@ -203,6 +246,21 @@ impl SysMaster { }); } + pub async fn do_heartbeat(master: Arc>) { + log::trace!("Starting heartbeat"); + let bcast = master.lock().await.broadcast(); + tokio::spawn(async move { + loop { + _ = time::sleep(Duration::from_secs(5)).await; + let mut p = MasterMessage::new(RequestType::Ping, "".to_string()); + let mut t = MinionTarget::new(); + t.add_hostname("*"); + p.add_target(t); + let _ = bcast.send(p.sendable().unwrap()); + } + }); + } + pub async fn do_outgoing(master: Arc>, tx: mpsc::Sender<(Vec, String)>) -> Result<(), SysinspectError> { log::trace!("Init outgoing channel"); let listener = master.lock().await.listener().await?; @@ -307,6 +365,8 @@ pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { // Accept connections and spawn tasks for each client SysMaster::do_outgoing(Arc::clone(&master), client_tx).await?; + SysMaster::do_heartbeat(Arc::clone(&master)).await; + // Listen for shutdown signal and cancel tasks tokio::signal::ctrl_c().await.expect("Failed to listen for ctrl_c"); log::info!("Received shutdown signal."); diff --git a/sysmaster/src/registry/mod.rs b/sysmaster/src/registry/mod.rs index 55f53d15..8c9b2b8a 100644 --- a/sysmaster/src/registry/mod.rs +++ b/sysmaster/src/registry/mod.rs @@ -4,6 +4,7 @@ Minion registry. It contains minion tasks, traits, location and other data pub mod mkb; pub mod rec; +pub mod session; use libsysinspect::SysinspectError; use rec::MinionRecord; diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 0423282f..0372ed18 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -3,10 +3,12 @@ use crate::{ proto, }; use libsysinspect::{ + proto::{errcodes::ProtoErrorCode, rqtypes::RequestType}, rsa, util::{self}, SysinspectError, }; +use once_cell::sync::Lazy; use std::{path::PathBuf, sync::Arc}; use tokio::net::TcpStream; use tokio::sync::Mutex; @@ -15,6 +17,10 @@ use tokio::{ io::{AsyncReadExt, BufReader}, sync::mpsc, }; +use uuid::Uuid; + +/// Session Id of the minion +pub static MINION_SID: Lazy = Lazy::new(|| Uuid::new_v4().to_string()); /// Talk-back to the master pub async fn request(stream: Arc>, msg: Vec) { @@ -33,7 +39,7 @@ pub async fn request(stream: Arc>, msg: Vec) { if let Err(e) = stm.flush().await { log::error!("Failed to flush writer to master: {}", e); } else { - log::debug!("To master: {}", String::from_utf8_lossy(&msg)); + log::trace!("To master: {}", String::from_utf8_lossy(&msg)); } } @@ -55,6 +61,7 @@ pub async fn minion(mut cfp: PathBuf, fingerprint: Option) -> Result<(), let wtsm_c = wstm.clone(); tokio::spawn(async move { let mut input = BufReader::new(rstm); + loop { let mut buff = [0u8; 4]; if let Err(e) = input.read_exact(&mut buff).await { @@ -69,49 +76,65 @@ pub async fn minion(mut cfp: PathBuf, fingerprint: Option) -> Result<(), break; } - match proto::msg::payload_to_msg(msg) { - Ok(msg) => { - log::debug!("Received: {:?}", msg); - match msg.req_type() { - libsysinspect::proto::rqtypes::RequestType::Add => { - log::debug!("Master accepts registration"); - } + let msg = match proto::msg::payload_to_msg(msg) { + Ok(msg) => msg, + Err(err) => { + log::error!("Error getting network payload as message: {err}"); + continue; + } + }; - libsysinspect::proto::rqtypes::RequestType::Reconnect => { - log::debug!("Master requires reconnection"); - log::info!("{}", msg.payload()); - std::process::exit(0); - } + log::trace!("Received: {:?}", msg); - libsysinspect::proto::rqtypes::RequestType::Remove => { - log::debug!("Master asks to unregister"); - } - libsysinspect::proto::rqtypes::RequestType::Command => { - log::debug!("Master sends a command"); - } - libsysinspect::proto::rqtypes::RequestType::Traits => { - log::debug!("Master requests traits"); - } - libsysinspect::proto::rqtypes::RequestType::AgentUnknown => { - let pbk_pem = msg.payload(); // Expected PEM RSA pub key - let (_, pbk) = rsa::keys::from_pem(None, Some(pbk_pem)).unwrap(); - let fpt = rsa::keys::get_fingerprint(&pbk.unwrap()).unwrap(); - - log::error!("Minion is not registered"); - log::info!("Master fingerprint: {}", fpt); - std::process::exit(1); + match msg.req_type() { + RequestType::Add => { + log::debug!("Master accepts registration"); + } + + RequestType::Reconnect => { + log::debug!("Master requires reconnection"); + log::info!("{}", msg.payload()); + std::process::exit(0); + } + + RequestType::Remove => { + log::debug!("Master asks to unregister"); + } + RequestType::Command => { + log::debug!("Master sends a command"); + match msg.get_retcode() { + ProtoErrorCode::AlreadyConnected => { + if MINION_SID.eq(msg.payload()) { + log::error!("Another minion from this machine is already connected"); + std::process::exit(1); + } } - _ => { - log::error!("Unknown request type"); + ret => { + log::debug!("Return code {:?} not yet implemented", ret); } } - - //request(wtsm_c.clone(), response).await; } - Err(err) => { - log::error!("{err}"); + RequestType::Traits => { + log::debug!("Master requests traits"); + } + RequestType::AgentUnknown => { + let pbk_pem = msg.payload(); // Expected PEM RSA pub key + let (_, pbk) = rsa::keys::from_pem(None, Some(pbk_pem)).unwrap(); + let fpt = rsa::keys::get_fingerprint(&pbk.unwrap()).unwrap(); + + log::error!("Minion is not registered"); + log::info!("Master fingerprint: {}", fpt); + std::process::exit(1); + } + RequestType::Ping => { + request(wtsm_c.clone(), proto::msg::get_pong()).await; + } + _ => { + log::error!("Unknown request type"); } } + + //request(wtsm_c.clone(), response).await; } }); diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index 90e26597..8ec9102a 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -1,5 +1,9 @@ pub mod msg { - use crate::{config::MinionConfig, minion::request, traits}; + use crate::{ + config::MinionConfig, + minion::{request, MINION_SID}, + traits, + }; use libsysinspect::{ proto::{rqtypes::RequestType, MinionMessage}, util::dataconv, @@ -11,12 +15,26 @@ pub mod msg { use std::sync::Arc; use tokio::{net::tcp::OwnedWriteHalf, sync::Mutex}; + /// Make pong message + pub fn get_pong() -> Vec { + let p = MinionMessage::new( + dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), + RequestType::Pong, + MINION_SID.to_string(), + ); + + if let Ok(data) = p.sendable() { + return data; + } + vec![] + } + /// Send ehlo pub async fn send_ehlo(stream: Arc>, cfg: MinionConfig) -> Result<(), SysinspectError> { let r = MinionMessage::new( dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), RequestType::Ehlo, - "".to_string(), + MINION_SID.to_string(), ); log::info!("Ehlo on {}", cfg.master()); From dd9e99e8bbbafded60732c744595dc5d6fcbd495 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 7 Nov 2024 22:36:57 +0100 Subject: [PATCH 068/148] Lintfix --- sysmaster/src/registry/session.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sysmaster/src/registry/session.rs b/sysmaster/src/registry/session.rs index b14a95fb..c966f814 100644 --- a/sysmaster/src/registry/session.rs +++ b/sysmaster/src/registry/session.rs @@ -47,6 +47,7 @@ impl SessionKeeper { } /// Collect the garbage (outdated sessions) + #[allow(clippy::useless_conversion)] // Not useless: it has to be a copy because it self-shooting itself fn gc(&mut self) { for s in self.sessions.keys().into_iter().map(|s| s.to_string()).collect::>() { self.alive(&s); @@ -61,7 +62,7 @@ impl SessionKeeper { /// Return uptime for a minion (seconds) pub fn uptime(&self, mid: &str) -> Option { - self.sessions.get(mid).and_then(|s| Some(s.uptime_sec())) + self.sessions.get(mid).map(|s| s.uptime_sec()) } /// Returns true if a minion is alive. @@ -83,6 +84,6 @@ impl SessionKeeper { /// Get session Id for the minion pub(crate) fn get_id(&self, mid: &str) -> Option { - self.sessions.get(mid).and_then(|s| Some(s.session_id())) + self.sessions.get(mid).map(|s| s.session_id()) } } From 50c613f3e54fe201aa8e6223778fd0adac5fef9a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 18:38:50 +0100 Subject: [PATCH 069/148] Update error code getter for MinionMessage --- libsysinspect/src/proto/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 82ea22cb..3cb344ec 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -98,6 +98,7 @@ impl MinionMessage { 2 => ProtoErrorCode::GeneralFailure, 3 => ProtoErrorCode::NotRegistered, 4 => ProtoErrorCode::AlreadyRegistered, + 5 => ProtoErrorCode::AlreadyConnected, _ => ProtoErrorCode::Unknown, } } From 5a43c3b38c97f9437c9f577e860adce6675b02be Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 18:38:59 +0100 Subject: [PATCH 070/148] Update protocol internal doc --- libsysinspect/src/proto/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/libsysinspect/src/proto/README.md b/libsysinspect/src/proto/README.md index cdcc8469..6cf345c5 100644 --- a/libsysinspect/src/proto/README.md +++ b/libsysinspect/src/proto/README.md @@ -117,6 +117,31 @@ The following request types for "`r`" are available: - `rsp` — A regular response to any command. - `ehlo` — Hello notice for a newly connected minion (any). Contains Minion Id RSA cipher. +## Types + +### Request/Response + +- `add` — Add a minion, registration request. +- `rm` — Remove a minion, un-registration. +- `rsp` — Regular response to any Master command. +- `cmd` — Regular command to any Minion. +- `tr` — Request to return all minion traits. +- `ehlo` — Hello message to initiate protocol. +- `retry` — Retry connect (e.g. after the registration). +- `pi` — Ping request. +- `po` — Pong response. +- `undef` — Unknown agent. + +### Return Codes + +- `Undef`: 0 — No specific return code or code is ignorable. +- `Success`: 1 — Successfully completed the routine. +- `GeneralFailure`: 2 — General failure, unspecified. Equal to 1 of POSIX. +- `NotRegistered`: 3 — Minion is not registered. Registration sequence required. +- `AlreadyRegistered`: 4 — Minion is already registered. +- `AlreadyConnected`: 5 — Minion connection duplicate. +- `Unknown`: N/A — Internal designator of unrecognised incoming error code. + ## Hello (ehlo) This sequence requires no established connection. From 5e96dafcc81cf4f3a8472aa973ab14ba3482188d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 18:39:20 +0100 Subject: [PATCH 071/148] Remove inspector from the main to its own incapsulable class --- src/inspector.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 59 ++++++----------------------------- 2 files changed, 91 insertions(+), 49 deletions(-) create mode 100644 src/inspector.rs diff --git a/src/inspector.rs b/src/inspector.rs new file mode 100644 index 00000000..115cae47 --- /dev/null +++ b/src/inspector.rs @@ -0,0 +1,81 @@ +use libsysinspect::{intp::actproc::response::ActionResponse, reactor::evtproc::EventProcessor}; + +#[derive(Debug, Default)] +pub struct SysInspectRunner { + model_pth: String, + state: Option, + entities: Vec, + + // Check book labels + cb_labels: Vec, +} + +impl SysInspectRunner { + pub fn new() -> SysInspectRunner { + SysInspectRunner { ..Default::default() } + } + + /// Set model path + pub fn set_model_path(&mut self, p: &str) { + self.model_pth = p.to_string() + } + + /// Set process state + pub fn set_state(&mut self, state: Option) { + self.state = state; + } + + /// Set entities to query + pub fn set_entities(&mut self, entities: Vec) { + self.entities = entities; + } + + /// Set checkbook labels + pub fn set_checkbook_labels(&mut self, labels: Vec) { + self.cb_labels = labels; + } + + pub fn start(&self) { + log::info!("Starting sysinspect runner"); + match libsysinspect::mdescr::mspec::load(&self.model_pth) { + Ok(spec) => { + log::debug!("Initalising inspector"); + match libsysinspect::intp::inspector::SysInspector::new(spec) { + Ok(isp) => { + // Setup event processor + let mut evtproc = EventProcessor::new().set_config(isp.cfg()); + + let actions = if !self.cb_labels.is_empty() { + isp.actions_by_relations(self.cb_labels.to_owned(), self.state.to_owned()) + } else { + isp.actions_by_entities(self.entities.to_owned(), self.state.to_owned()) + }; + + match actions { + Ok(actions) => { + for ac in actions { + match ac.run() { + Ok(response) => { + let response = response.unwrap_or(ActionResponse::default()); + evtproc.receiver().register(response.eid().to_owned(), response); + } + Err(err) => { + log::error!("{err}") + } + } + } + evtproc.process(); + } + Err(err) => { + log::error!("{}", err); + } + } + } + Err(err) => log::error!("{err}"), + } + log::debug!("Done"); + } + Err(err) => log::error!("Error: {}", err), + }; + } +} diff --git a/src/main.rs b/src/main.rs index e8ed6a73..ea5dc5a8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,20 +1,18 @@ use colored::Colorize; -use libsysinspect::{ - intp::actproc::response::ActionResponse, - logger, - reactor::{evtproc::EventProcessor, handlers}, -}; +use inspector::SysInspectRunner; +use libsysinspect::{logger, reactor::handlers}; use log::LevelFilter; use std::env; mod clidef; +mod inspector; mod mcf; static VERSION: &str = "0.2.0"; static LOGGER: logger::STDOUTLogger = logger::STDOUTLogger; /// Display event handlers -fn print_event_handlers() { +pub fn print_event_handlers() { handlers::registry::init_handlers(); println!("{}", format!("Supported event handlers in {}:", clidef::APPNAME.bold()).yellow()); for (i, h) in handlers::registry::get_handler_names().iter().enumerate() { @@ -67,48 +65,11 @@ fn main() { } if let Some(mpath) = params.get_one::("model") { - match libsysinspect::mdescr::mspec::load(mpath) { - Ok(spec) => { - log::debug!("Initalising inspector"); - match libsysinspect::intp::inspector::SysInspector::new(spec) { - Ok(isp) => { - // Setup event processor - let mut evtproc = EventProcessor::new().set_config(isp.cfg()); - - let arg_state = params.get_one::("state").cloned(); - let arg_labels = clidef::split_by(¶ms, "labels", None); - - let actions = if !arg_labels.is_empty() { - isp.actions_by_relations(arg_labels, arg_state.to_owned()) - } else { - isp.actions_by_entities(clidef::split_by(¶ms, "entities", None), arg_state) - }; - - match actions { - Ok(actions) => { - for ac in actions { - match ac.run() { - Ok(response) => { - let response = response.unwrap_or(ActionResponse::default()); - evtproc.receiver().register(response.eid().to_owned(), response); - } - Err(err) => { - log::error!("{err}") - } - } - } - evtproc.process(); - } - Err(err) => { - log::error!("{}", err); - } - } - } - Err(err) => log::error!("{err}"), - } - log::debug!("Done"); - } - Err(err) => log::error!("Error: {}", err), - }; + let mut sr = SysInspectRunner::new(); + sr.set_model_path(mpath); + sr.set_state(params.get_one::("state").cloned()); + sr.set_entities(clidef::split_by(¶ms, "entities", None)); + sr.set_checkbook_labels(clidef::split_by(¶ms, "labels", None)); + sr.start(); } } From 2c951395a43724a02dd40f361bf9d8f877c32134 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 19:31:23 +0100 Subject: [PATCH 072/148] Move inspector app to libsysinspect for common use --- {src => libsysinspect/src}/inspector.rs | 11 ++++++++--- libsysinspect/src/lib.rs | 1 + src/main.rs | 1 - 3 files changed, 9 insertions(+), 4 deletions(-) rename {src => libsysinspect/src}/inspector.rs (91%) diff --git a/src/inspector.rs b/libsysinspect/src/inspector.rs similarity index 91% rename from src/inspector.rs rename to libsysinspect/src/inspector.rs index 115cae47..a3175f41 100644 --- a/src/inspector.rs +++ b/libsysinspect/src/inspector.rs @@ -1,4 +1,9 @@ -use libsysinspect::{intp::actproc::response::ActionResponse, reactor::evtproc::EventProcessor}; +use crate::{ + intp::{self, inspector::SysInspector}, + mdescr::mspec, + reactor::evtproc::EventProcessor, +}; +use intp::actproc::response::ActionResponse; #[derive(Debug, Default)] pub struct SysInspectRunner { @@ -37,10 +42,10 @@ impl SysInspectRunner { pub fn start(&self) { log::info!("Starting sysinspect runner"); - match libsysinspect::mdescr::mspec::load(&self.model_pth) { + match mspec::load(&self.model_pth) { Ok(spec) => { log::debug!("Initalising inspector"); - match libsysinspect::intp::inspector::SysInspector::new(spec) { + match SysInspector::new(spec) { Ok(isp) => { // Setup event processor let mut evtproc = EventProcessor::new().set_config(isp.cfg()); diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 5fda82fc..35d390ea 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -7,6 +7,7 @@ use std::{ use mdescr::mspec; +pub mod inspector; pub mod intp; pub mod logger; pub mod mdescr; diff --git a/src/main.rs b/src/main.rs index ea5dc5a8..783af717 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,6 @@ use log::LevelFilter; use std::env; mod clidef; -mod inspector; mod mcf; static VERSION: &str = "0.2.0"; From b8504c445774ff3f5a0feaa56c927b4ccd4c9d94 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 19:31:45 +0100 Subject: [PATCH 073/148] Remove dead prop code --- src/mcf/README.md | 22 ---------------------- src/mcf/cfread.rs | 0 src/mcf/mod.rs | 1 - 3 files changed, 23 deletions(-) delete mode 100644 src/mcf/README.md delete mode 100644 src/mcf/cfread.rs delete mode 100644 src/mcf/mod.rs diff --git a/src/mcf/README.md b/src/mcf/README.md deleted file mode 100644 index ad5bf45a..00000000 --- a/src/mcf/README.md +++ /dev/null @@ -1,22 +0,0 @@ -Model design - -A model is a directory with the following layout: - -``` - -| -+-- entities/ -| // Here are all entities -| -+-- relations/ -| // Here are all relations between entities -| -+-- actions/ -| // Here are all actions on each relation -| -+-- constraints/ -| // Here are all constraints for each entity -| -+-- model.cfg - // Main model description -``` diff --git a/src/mcf/cfread.rs b/src/mcf/cfread.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/src/mcf/mod.rs b/src/mcf/mod.rs deleted file mode 100644 index 636e8144..00000000 --- a/src/mcf/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod cfread; From e33c479d70f304612c03c47a3172555ae6fe065e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 19:32:12 +0100 Subject: [PATCH 074/148] Update main to extend on sysinspect network use --- src/clidef.rs | 24 ++++++++++++++++++++++++ src/main.rs | 6 ++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/clidef.rs b/src/clidef.rs index 708357ac..f1f2ae63 100644 --- a/src/clidef.rs +++ b/src/clidef.rs @@ -17,6 +17,24 @@ pub fn cli(version: &'static str) -> Command { .about(format!("{} - {}", APPNAME.bright_magenta().bold(), "is a tool for anomaly detection and root cause analysis in a known system")) .override_usage(format!("{} [OPTIONS] [FILTERS]", APPNAME)) + // Sysinspect + .next_help_heading("Main") + .arg( + Arg::new("query") + .help("Network query") + .required(false) + .index(1) + ) + .arg( + Arg::new("traits") + .short('t') + .long("traits") + .help("Specify traits to select remote systems") + ) + + // Local + .next_help_heading("Local") + // Config .arg( Arg::new("model") @@ -56,6 +74,12 @@ pub fn cli(version: &'static str) -> Command { // Other .next_help_heading("Other") + .arg( + Arg::new("config") + .short('c') + .long("config") + .help("Specify alternative configuration") + ) .arg( Arg::new("debug") .short('d') diff --git a/src/main.rs b/src/main.rs index 783af717..fe04f3f8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,15 @@ use colored::Colorize; -use inspector::SysInspectRunner; -use libsysinspect::{logger, reactor::handlers}; +use libsysinspect::{inspector::SysInspectRunner, logger, reactor::handlers}; use log::LevelFilter; use std::env; mod clidef; -mod mcf; static VERSION: &str = "0.2.0"; static LOGGER: logger::STDOUTLogger = logger::STDOUTLogger; /// Display event handlers -pub fn print_event_handlers() { +fn print_event_handlers() { handlers::registry::init_handlers(); println!("{}", format!("Supported event handlers in {}:", clidef::APPNAME.bold()).yellow()); for (i, h) in handlers::registry::get_handler_names().iter().enumerate() { From a30fd3a7f07f120af3f008c0baf42669491252c2 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 20:14:52 +0100 Subject: [PATCH 075/148] Move network configs to one common place --- libsysinspect/src/{util/cfg.rs => cfg/mod.rs} | 0 libsysinspect/src/lib.rs | 1 + libsysinspect/src/mdescr/mspec.rs | 2 +- libsysinspect/src/util/mod.rs | 1 - sysmaster/src/main.rs | 2 +- sysminion/src/minion.rs | 3 ++- 6 files changed, 5 insertions(+), 4 deletions(-) rename libsysinspect/src/{util/cfg.rs => cfg/mod.rs} (100%) diff --git a/libsysinspect/src/util/cfg.rs b/libsysinspect/src/cfg/mod.rs similarity index 100% rename from libsysinspect/src/util/cfg.rs rename to libsysinspect/src/cfg/mod.rs diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 35d390ea..4cb92d6e 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -7,6 +7,7 @@ use std::{ use mdescr::mspec; +pub mod cfg; pub mod inspector; pub mod intp; pub mod logger; diff --git a/libsysinspect/src/mdescr/mspec.rs b/libsysinspect/src/mdescr/mspec.rs index 4fc0a23e..05b89563 100644 --- a/libsysinspect/src/mdescr/mspec.rs +++ b/libsysinspect/src/mdescr/mspec.rs @@ -1,5 +1,5 @@ use super::{datapatch, mspecdef::ModelSpec}; -use crate::{util::cfg::select_config, SysinspectError}; +use crate::{cfg::select_config, SysinspectError}; use serde_yaml::Value; use std::{ fs::{self}, diff --git a/libsysinspect/src/util/mod.rs b/libsysinspect/src/util/mod.rs index 3d9b2b17..77c2b205 100644 --- a/libsysinspect/src/util/mod.rs +++ b/libsysinspect/src/util/mod.rs @@ -1,2 +1 @@ -pub mod cfg; pub mod dataconv; diff --git a/sysmaster/src/main.rs b/sysmaster/src/main.rs index 6f795011..d84d6611 100644 --- a/sysmaster/src/main.rs +++ b/sysmaster/src/main.rs @@ -5,7 +5,7 @@ mod registry; mod rmt; use clidef::cli; -use libsysinspect::{logger, util::cfg::select_config, SysinspectError}; +use libsysinspect::{cfg::select_config, logger, SysinspectError}; use log::LevelFilter; use rmt::send_message; use std::{env, path::PathBuf}; diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 0372ed18..4a684260 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -3,6 +3,7 @@ use crate::{ proto, }; use libsysinspect::{ + cfg, proto::{errcodes::ProtoErrorCode, rqtypes::RequestType}, rsa, util::{self}, @@ -47,7 +48,7 @@ pub async fn request(stream: Arc>, msg: Vec) { pub async fn minion(mut cfp: PathBuf, fingerprint: Option) -> Result<(), SysinspectError> { let fingerprint = fingerprint.unwrap_or_default(); if !cfp.exists() { - cfp = util::cfg::select_config()?; + cfp = cfg::select_config()?; } let cfg = config::MinionConfig::new(cfp)?; //let st = traits::get_traits(); From 84a79719983825adacfeda0fbaf983eee8151442 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 21:04:47 +0100 Subject: [PATCH 076/148] Move master and minion internal config structures to the mmconfig --- .../src/cfg/mmconf.rs | 31 ++++++++++++++++- libsysinspect/src/cfg/mod.rs | 2 ++ sysmaster/src/main.rs | 8 +++-- sysmaster/src/master.rs | 10 +++--- sysminion/src/config.rs | 33 ------------------- sysminion/src/main.rs | 1 - sysminion/src/minion.rs | 13 +++----- sysminion/src/proto.rs | 2 +- 8 files changed, 46 insertions(+), 54 deletions(-) rename sysmaster/src/config.rs => libsysinspect/src/cfg/mmconf.rs (61%) delete mode 100644 sysminion/src/config.rs diff --git a/sysmaster/src/config.rs b/libsysinspect/src/cfg/mmconf.rs similarity index 61% rename from sysmaster/src/config.rs rename to libsysinspect/src/cfg/mmconf.rs index 9c142f8c..1f2bdb63 100644 --- a/sysmaster/src/config.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -1,4 +1,4 @@ -use libsysinspect::{intp::functions::get_by_namespace, SysinspectError}; +use crate::{intp::functions::get_by_namespace, SysinspectError}; use serde::{Deserialize, Serialize}; use serde_yaml::{from_str, from_value, Value}; use std::{fs, path::PathBuf}; @@ -7,6 +7,35 @@ static DEFAULT_ADDR: &str = "0.0.0.0"; static DEFAULT_PORT: u32 = 4200; static DEFAULT_SOCKET: &str = "/var/run/sysinspect-master.socket"; +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +pub struct MinionConfig { + #[serde(rename = "master.ip")] + master_ip: String, + + #[serde(rename = "master.port")] + master_port: u32, +} + +impl MinionConfig { + pub fn new(p: PathBuf) -> Result { + let cp = p.as_os_str().to_str().unwrap_or_default(); + if !p.exists() { + return Err(SysinspectError::ConfigError(format!("File not found: {}", cp))); + } + + if let Some(cfgv) = get_by_namespace(Some(from_str::(&fs::read_to_string(&p)?)?), "config.minion") { + return Ok(from_value::(cfgv)?); + } + + Err(SysinspectError::ConfigError(format!("Unable to read config at: {}", cp))) + } + + /// Return master addr + pub fn master(&self) -> String { + format!("{}:{}", self.master_ip, self.master_port) + } +} + #[derive(Debug, Serialize, Deserialize, Default, Clone)] pub struct MasterConfig { // Bind IP listener. Default "the world", i.e. 0.0.0.0 diff --git a/libsysinspect/src/cfg/mod.rs b/libsysinspect/src/cfg/mod.rs index dd4f301a..eaed6084 100644 --- a/libsysinspect/src/cfg/mod.rs +++ b/libsysinspect/src/cfg/mod.rs @@ -2,6 +2,8 @@ Config reader */ +pub mod mmconf; + use crate::SysinspectError; use nix::unistd::Uid; use std::{env, path::PathBuf}; diff --git a/sysmaster/src/main.rs b/sysmaster/src/main.rs index d84d6611..a87f6915 100644 --- a/sysmaster/src/main.rs +++ b/sysmaster/src/main.rs @@ -1,11 +1,13 @@ mod clidef; -mod config; mod master; mod registry; mod rmt; use clidef::cli; -use libsysinspect::{cfg::select_config, logger, SysinspectError}; +use libsysinspect::{ + cfg::{mmconf::MasterConfig, select_config}, + logger, SysinspectError, +}; use log::LevelFilter; use rmt::send_message; use std::{env, path::PathBuf}; @@ -56,7 +58,7 @@ async fn main() -> Result<(), SysinspectError> { } }; } - let cfg = config::MasterConfig::new(cfp)?; + let cfg = MasterConfig::new(cfp)?; // Mode let query = params.get_one::("query").unwrap_or(&"".to_string()).to_owned(); diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 94399b48..d11b0598 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -1,11 +1,9 @@ -use crate::{ - config::MasterConfig, - registry::{ - mkb::MinionsKeyRegistry, - session::{self, SessionKeeper}, - }, +use crate::registry::{ + mkb::MinionsKeyRegistry, + session::{self, SessionKeeper}, }; use libsysinspect::{ + cfg::mmconf::MasterConfig, proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; diff --git a/sysminion/src/config.rs b/sysminion/src/config.rs deleted file mode 100644 index 2bdad964..00000000 --- a/sysminion/src/config.rs +++ /dev/null @@ -1,33 +0,0 @@ -use libsysinspect::{intp::functions::get_by_namespace, SysinspectError}; -use serde::{Deserialize, Serialize}; -use serde_yaml::{from_str, from_value, Value}; -use std::{fs, path::PathBuf}; - -#[derive(Debug, Serialize, Deserialize, Default, Clone)] -pub struct MinionConfig { - #[serde(rename = "master.ip")] - master_ip: String, - - #[serde(rename = "master.port")] - master_port: u32, -} - -impl MinionConfig { - pub fn new(p: PathBuf) -> Result { - let cp = p.as_os_str().to_str().unwrap_or_default(); - if !p.exists() { - return Err(SysinspectError::ConfigError(format!("File not found: {}", cp))); - } - - if let Some(cfgv) = get_by_namespace(Some(from_str::(&fs::read_to_string(&p)?)?), "config.minion") { - return Ok(from_value::(cfgv)?); - } - - Err(SysinspectError::ConfigError(format!("Unable to read config at: {}", cp))) - } - - /// Return master addr - pub fn master(&self) -> String { - format!("{}:{}", self.master_ip, self.master_port) - } -} diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 620dc5a3..07249e94 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -1,5 +1,4 @@ mod clidef; -mod config; mod minion; mod proto; mod rsa; diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 4a684260..60b5e2e7 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,13 +1,8 @@ -use crate::{ - config::{self}, - proto, -}; +use crate::proto; use libsysinspect::{ - cfg, + cfg::{self, mmconf::MinionConfig}, proto::{errcodes::ProtoErrorCode, rqtypes::RequestType}, - rsa, - util::{self}, - SysinspectError, + rsa, SysinspectError, }; use once_cell::sync::Lazy; use std::{path::PathBuf, sync::Arc}; @@ -50,7 +45,7 @@ pub async fn minion(mut cfp: PathBuf, fingerprint: Option) -> Result<(), if !cfp.exists() { cfp = cfg::select_config()?; } - let cfg = config::MinionConfig::new(cfp)?; + let cfg = MinionConfig::new(cfp)?; //let st = traits::get_traits(); let mkeys = crate::rsa::MinionRSAKeyManager::new(None)?; // XXX: Get optional root from the configuration diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index 8ec9102a..2f41571e 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -1,10 +1,10 @@ pub mod msg { use crate::{ - config::MinionConfig, minion::{request, MINION_SID}, traits, }; use libsysinspect::{ + cfg::mmconf::MinionConfig, proto::{rqtypes::RequestType, MinionMessage}, util::dataconv, }; From 0efbf4a818a3101af92781c122e5c8576848658a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 21:09:58 +0100 Subject: [PATCH 077/148] Allow override default config selector --- libsysinspect/src/cfg/mod.rs | 9 ++++++++- libsysinspect/src/mdescr/mspec.rs | 2 +- sysmaster/src/main.rs | 2 +- sysminion/src/minion.rs | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libsysinspect/src/cfg/mod.rs b/libsysinspect/src/cfg/mod.rs index eaed6084..9fc9e23c 100644 --- a/libsysinspect/src/cfg/mod.rs +++ b/libsysinspect/src/cfg/mod.rs @@ -12,7 +12,14 @@ pub const APP_CONF: &str = "sysinspect.conf"; pub const APP_DOTCONF: &str = ".sysinspect"; /// Select app conf -pub fn select_config() -> Result { +pub fn select_config(p: Option) -> Result { + // Override path from options + if let Some(ovrp) = p { + if ovrp.exists() { + return Ok(ovrp); + } + } + // Current let cfp: PathBuf = env::current_dir()?.canonicalize()?.join(APP_CONF); if cfp.exists() { diff --git a/libsysinspect/src/mdescr/mspec.rs b/libsysinspect/src/mdescr/mspec.rs index 05b89563..758a10d9 100644 --- a/libsysinspect/src/mdescr/mspec.rs +++ b/libsysinspect/src/mdescr/mspec.rs @@ -122,7 +122,7 @@ impl SpecLoader { } // Load app config and merge to the main model - base.push(serde_yaml::from_str::(&fs::read_to_string(select_config()?)?)?); + base.push(serde_yaml::from_str::(&fs::read_to_string(select_config(None)?)?)?); let mut base = self.merge_parts(&mut base)?; if !iht.is_empty() { diff --git a/sysmaster/src/main.rs b/sysmaster/src/main.rs index a87f6915..4ffbb795 100644 --- a/sysmaster/src/main.rs +++ b/sysmaster/src/main.rs @@ -47,7 +47,7 @@ async fn main() -> Result<(), SysinspectError> { // Get config let mut cfp = PathBuf::from(params.get_one::("config").unwrap_or(&"".to_string()).to_owned()); if !cfp.exists() { - cfp = match select_config() { + cfp = match select_config(None) { Ok(cfp) => { log::debug!("Reading config at {}", cfp.to_str().unwrap_or_default()); cfp diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 60b5e2e7..d9f55af2 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -43,7 +43,7 @@ pub async fn request(stream: Arc>, msg: Vec) { pub async fn minion(mut cfp: PathBuf, fingerprint: Option) -> Result<(), SysinspectError> { let fingerprint = fingerprint.unwrap_or_default(); if !cfp.exists() { - cfp = cfg::select_config()?; + cfp = cfg::select_config(None)?; } let cfg = MinionConfig::new(cfp)?; //let st = traits::get_traits(); From 8260822aa2e2fa3fcefd74135d7b419ccdfbf40f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 21:21:18 +0100 Subject: [PATCH 078/148] Refactor sysinspect, incorporating remote command --- libsysinspect/src/cfg/mod.rs | 3 +- src/main.rs | 64 ++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/libsysinspect/src/cfg/mod.rs b/libsysinspect/src/cfg/mod.rs index 9fc9e23c..285f7d66 100644 --- a/libsysinspect/src/cfg/mod.rs +++ b/libsysinspect/src/cfg/mod.rs @@ -12,9 +12,10 @@ pub const APP_CONF: &str = "sysinspect.conf"; pub const APP_DOTCONF: &str = ".sysinspect"; /// Select app conf -pub fn select_config(p: Option) -> Result { +pub fn select_config(p: Option) -> Result { // Override path from options if let Some(ovrp) = p { + let ovrp = PathBuf::from(ovrp); if ovrp.exists() { return Ok(ovrp); } diff --git a/src/main.rs b/src/main.rs index fe04f3f8..f6d1de53 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,14 @@ +use clap::ArgMatches; use colored::Colorize; -use libsysinspect::{inspector::SysInspectRunner, logger, reactor::handlers}; +use libsysinspect::{ + cfg::{mmconf::MasterConfig, select_config}, + inspector::SysInspectRunner, + logger, + reactor::handlers, + SysinspectError, +}; use log::LevelFilter; -use std::env; +use std::{env, fs::OpenOptions, io::Write, path::PathBuf}; mod clidef; @@ -18,6 +25,32 @@ fn print_event_handlers() { println!(); } +/// Call master via FIFO +fn call_master_fifo(msg: &str, fifo: &str) -> Result<(), SysinspectError> { + OpenOptions::new().write(true).open(fifo)?.write_all(format!("{}\n", msg).as_bytes())?; + + log::debug!("Message sent to FIFO: {}", msg); + Ok(()) +} + +/// Set logger +fn set_logger(p: &ArgMatches) { + if let Err(err) = log::set_logger(&LOGGER).map(|()| { + log::set_max_level(match p.get_count("debug") { + 0 => LevelFilter::Info, + 1 => LevelFilter::Debug, + 2.. => LevelFilter::max(), + }) + }) { + println!("{}", err) + } +} + +/// Get configuration of the master +fn get_cfg(p: &ArgMatches) -> Result { + MasterConfig::new(select_config(p.get_one::("config").cloned())?) +} + fn main() { let args: Vec = env::args().collect(); let mut cli = clidef::cli(VERSION); @@ -31,6 +64,9 @@ fn main() { // Our main params let params = cli.to_owned().get_matches(); + // Set logger + set_logger(¶ms); + // Print help? if *params.get_one::("help").unwrap() { return { @@ -50,18 +86,20 @@ fn main() { return; } - // Setup logger - if let Err(err) = log::set_logger(&LOGGER).map(|()| { - log::set_max_level(match params.get_count("debug") { - 0 => LevelFilter::Info, - 1 => LevelFilter::Debug, - 2.. => LevelFilter::max(), - }) - }) { - return println!("{}", err); - } + // Get master config + let cfg = match get_cfg(¶ms) { + Ok(cfg) => cfg, + Err(err) => { + log::error!("Unable to get master configuration: {err}"); + std::process::exit(1); + } + }; - if let Some(mpath) = params.get_one::("model") { + if let Some(query) = params.get_one::("query") { + if let Err(err) = call_master_fifo(&query, &cfg.socket()) { + log::error!("Cannot reach master: {err}"); + } + } else if let Some(mpath) = params.get_one::("model") { let mut sr = SysInspectRunner::new(); sr.set_model_path(mpath); sr.set_state(params.get_one::("state").cloned()); From b680e3446fdb3b73af8ece694d62e4381ff54941 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 8 Nov 2024 21:21:52 +0100 Subject: [PATCH 079/148] Lintfix --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index f6d1de53..6771f48e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use libsysinspect::{ SysinspectError, }; use log::LevelFilter; -use std::{env, fs::OpenOptions, io::Write, path::PathBuf}; +use std::{env, fs::OpenOptions, io::Write}; mod clidef; @@ -96,7 +96,7 @@ fn main() { }; if let Some(query) = params.get_one::("query") { - if let Err(err) = call_master_fifo(&query, &cfg.socket()) { + if let Err(err) = call_master_fifo(query, &cfg.socket()) { log::error!("Cannot reach master: {err}"); } } else if let Some(mpath) = params.get_one::("model") { From c7f65af5d660e279f16e363a8d94e4a8f79ee799 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 16:36:47 +0100 Subject: [PATCH 080/148] Update dependencies --- Cargo.lock | 806 ++++++++++++++++++++++++++++++++++++++++++- sysmaster/Cargo.toml | 1 + 2 files changed, 805 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 354995b5..8fb52fb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,189 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "actix-codec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +dependencies = [ + "bitflags 2.6.0", + "bytes", + "futures-core", + "futures-sink", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "actix-http" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48f96fc3003717aeb9856ca3d02a8c7de502667ad76eeacd830b48d2e91fac4" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "ahash", + "base64", + "bitflags 2.6.0", + "brotli", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "h2", + "http", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "mime", + "percent-encoding", + "pin-project-lite", + "rand", + "sha1", + "smallvec", + "tokio", + "tokio-util", + "tracing", + "zstd", +] + +[[package]] +name = "actix-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +dependencies = [ + "quote", + "syn 2.0.85", +] + +[[package]] +name = "actix-router" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" +dependencies = [ + "bytestring", + "cfg-if", + "http", + "regex", + "regex-lite", + "serde", + "tracing", +] + +[[package]] +name = "actix-rt" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" +dependencies = [ + "futures-core", + "tokio", +] + +[[package]] +name = "actix-server" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca2549781d8dd6d75c40cf6b6051260a2cc2f3c62343d761a969a0640646894" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" +dependencies = [ + "futures-core", + "paste", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9180d76e5cc7ccbc4d60a506f2c727730b154010262df5b910eb17dbe4b8cb38" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "ahash", + "bytes", + "bytestring", + "cfg-if", + "cookie", + "derive_more", + "encoding_rs", + "futures-core", + "futures-util", + "impl-more", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "regex-lite", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" +dependencies = [ + "actix-router", + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "addr2line" version = "0.24.2" @@ -28,6 +211,19 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -37,6 +233,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -223,6 +434,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -241,6 +473,15 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "bytestring" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +dependencies = [ + "bytes", +] + [[package]] name = "cbc" version = "0.1.2" @@ -375,6 +616,23 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cookie" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -493,6 +751,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "0.99.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.85", +] + [[package]] name = "digest" version = "0.10.7" @@ -526,6 +806,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "dunce" version = "1.0.5" @@ -572,6 +863,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -604,6 +904,21 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fs2" version = "0.4.3" @@ -757,6 +1072,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.0" @@ -799,6 +1133,29 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hyphenation" version = "0.8.4" @@ -845,6 +1202,151 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "impl-more" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" + [[package]] name = "indexmap" version = "2.6.0" @@ -944,6 +1446,12 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + [[package]] name = "lazy_static" version = "1.5.0" @@ -1028,6 +1536,29 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + +[[package]] +name = "local-channel" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +dependencies = [ + "futures-core", + "futures-sink", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" + [[package]] name = "lock_api" version = "0.4.12" @@ -1071,6 +1602,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1094,6 +1631,7 @@ checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi 0.3.9", "libc", + "log", "wasi", "windows-sys 0.52.0", ] @@ -1193,6 +1731,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1321,6 +1865,12 @@ dependencies = [ "base64ct", ] +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pin-project-lite" version = "0.2.15" @@ -1371,12 +1921,24 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "pocket-resources" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c135f38778ad324d9e9ee68690bac2c1a51f340fdf96ca13e2ab3914eb2e51d8" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1567,6 +2129,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -1796,6 +2364,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -1929,6 +2509,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" @@ -1963,6 +2549,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "sysinfo" version = "0.31.4" @@ -2007,6 +2604,7 @@ dependencies = [ name = "sysmaster" version = "0.1.0" dependencies = [ + "actix-web", "clap", "colored", "ed25519-dalek", @@ -2104,11 +2702,52 @@ dependencies = [ "syn 2.0.85", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tokio" -version = "1.41.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", @@ -2144,6 +2783,39 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.17.0" @@ -2186,6 +2858,29 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "url" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2531,6 +3226,42 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2552,6 +3283,27 @@ dependencies = [ "syn 2.0.85", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" @@ -2571,3 +3323,53 @@ dependencies = [ "quote", "syn 2.0.85", ] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/sysmaster/Cargo.toml b/sysmaster/Cargo.toml index dd20555a..701b4323 100644 --- a/sysmaster/Cargo.toml +++ b/sysmaster/Cargo.toml @@ -32,3 +32,4 @@ log = "0.4.22" sled = "0.34.7" rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } uuid = { version = "1.11.0", features = ["v4"] } +actix-web = "4.9.0" From 234e55daee4079d6fa119c60b840c6c61fa9c1fc Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 16:37:34 +0100 Subject: [PATCH 081/148] Add fileserver config --- sysinspect.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sysinspect.conf b/sysinspect.conf index 85ad5ede..3e4810b2 100644 --- a/sysinspect.conf +++ b/sysinspect.conf @@ -8,6 +8,9 @@ config: bind.ip: 0.0.0.0 bind.port: 4200 + fileserver.bind.ip: 0.0.0.0 + fileserver.bind.port: 4201 + # Configuration that is present only on a minion node minion: master.ip: 192.168.2.102 From 20bd0806f14e4d73db964257f2b3ad5abef2f280 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 16:37:55 +0100 Subject: [PATCH 082/148] Update configuration struct for fileserver options --- libsysinspect/src/cfg/mmconf.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libsysinspect/src/cfg/mmconf.rs b/libsysinspect/src/cfg/mmconf.rs index 1f2bdb63..b748cdf2 100644 --- a/libsysinspect/src/cfg/mmconf.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -5,6 +5,7 @@ use std::{fs, path::PathBuf}; static DEFAULT_ADDR: &str = "0.0.0.0"; static DEFAULT_PORT: u32 = 4200; +static DEFAULT_FILESERVER_PORT: u32 = 4201; static DEFAULT_SOCKET: &str = "/var/run/sysinspect-master.socket"; #[derive(Debug, Serialize, Deserialize, Default, Clone)] @@ -48,6 +49,12 @@ pub struct MasterConfig { // Path to FIFO socket. Default: /var/run/sysinspect-master.socket socket: Option, + + #[serde(rename = "fileserver.bind.ip")] + fileserver_ip: Option, + + #[serde(rename = "fileserver.bind.port")] + fileserver_port: Option, } impl MasterConfig { @@ -73,4 +80,13 @@ impl MasterConfig { pub fn socket(&self) -> String { self.socket.to_owned().unwrap_or(DEFAULT_SOCKET.to_string()) } + + /// Return fileserver addr + pub fn fileserver_bind_addr(&self) -> String { + format!( + "{}:{}", + self.fileserver_ip.to_owned().unwrap_or(DEFAULT_ADDR.to_string()), + self.fileserver_port.unwrap_or(DEFAULT_FILESERVER_PORT) + ) + } } From b1064a3d47462a466bcc8cb0e3c3f698db8af91b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 16:38:17 +0100 Subject: [PATCH 083/148] Implement a basic HTTP-based fileserver --- sysmaster/src/dataserv/fls.rs | 47 +++++++++++++++++++++++++++++++++++ sysmaster/src/dataserv/mod.rs | 1 + sysmaster/src/main.rs | 4 ++- sysmaster/src/master.rs | 14 ++++++++--- 4 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 sysmaster/src/dataserv/fls.rs create mode 100644 sysmaster/src/dataserv/mod.rs diff --git a/sysmaster/src/dataserv/fls.rs b/sysmaster/src/dataserv/fls.rs new file mode 100644 index 00000000..1d7b4d91 --- /dev/null +++ b/sysmaster/src/dataserv/fls.rs @@ -0,0 +1,47 @@ +use actix_web::{rt::System, web, App, HttpResponse, HttpServer, Responder}; +use libsysinspect::{cfg::mmconf::MasterConfig, SysinspectError}; +use std::{ + fs, + path::{Path, PathBuf}, + thread, +}; + +use crate::registry::CFG_DEFAULT_ROOT; +static FILESERVER_ROOT_DIR: &str = "data"; + +// Separate handler on every HTTP call +async fn serve_file(path: web::Path, _cfg: web::Data) -> impl Responder { + let pth = Path::new(CFG_DEFAULT_ROOT).join(FILESERVER_ROOT_DIR).join(path.into_inner()); + log::debug!("Requested local file: {:?}", pth); + if pth.is_file() { + return HttpResponse::Ok().body(fs::read(pth).unwrap()); + } + log::error!("File {:?} was not found", pth); + HttpResponse::NotFound().body("File not found") +} + +/// Start fileserver +pub async fn start(cfg: MasterConfig) -> Result<(), SysinspectError> { + thread::spawn(move || { + let c_cfg = cfg.clone(); + System::new().block_on(async move { + let server = HttpServer::new(move || { + App::new().app_data(web::Data::new(cfg.clone())).service(web::resource("/{path:.*}").to(serve_file)) + }) + .bind(c_cfg.fileserver_bind_addr()); + + match server { + Ok(server) => { + if let Err(err) = server.run().await { + Err(err) + } else { + Ok(()) + } + } + Err(err) => Err(err), + } + }) + }); + + Ok(()) +} diff --git a/sysmaster/src/dataserv/mod.rs b/sysmaster/src/dataserv/mod.rs new file mode 100644 index 00000000..5d975762 --- /dev/null +++ b/sysmaster/src/dataserv/mod.rs @@ -0,0 +1 @@ +pub mod fls; diff --git a/sysmaster/src/main.rs b/sysmaster/src/main.rs index 4ffbb795..a6fbb8c2 100644 --- a/sysmaster/src/main.rs +++ b/sysmaster/src/main.rs @@ -1,4 +1,5 @@ mod clidef; +mod dataserv; mod master; mod registry; mod rmt; @@ -10,7 +11,8 @@ use libsysinspect::{ }; use log::LevelFilter; use rmt::send_message; -use std::{env, path::PathBuf}; +use std::env; +use std::path::PathBuf; static APPNAME: &str = "sysmaster"; static VERSION: &str = "0.0.1"; diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index d11b0598..06d8855e 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -1,6 +1,9 @@ -use crate::registry::{ - mkb::MinionsKeyRegistry, - session::{self, SessionKeeper}, +use crate::{ + dataserv::fls, + registry::{ + mkb::MinionsKeyRegistry, + session::{self, SessionKeeper}, + }, }; use libsysinspect::{ cfg::mmconf::MasterConfig, @@ -346,7 +349,7 @@ impl SysMaster { } pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { - let master = Arc::new(Mutex::new(SysMaster::new(cfg)?)); + let master = Arc::new(Mutex::new(SysMaster::new(cfg.to_owned())?)); { let mut m = master.lock().await; m.init().await?; @@ -354,6 +357,9 @@ pub(crate) async fn master(cfg: MasterConfig) -> Result<(), SysinspectError> { let (client_tx, client_rx) = mpsc::channel::<(Vec, String)>(100); + // Start internal fileserver for minions + fls::start(cfg).await?; + // Task to read from the FIFO and broadcast messages to clients SysMaster::do_fifo(Arc::clone(&master)).await; From 533823a7e0116b364a7df61a7f63c86b5368bac0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 22:09:06 +0100 Subject: [PATCH 084/148] Update deps --- Cargo.lock | 455 ++++++++++++++++++++++++++++++++++++++++++- sysminion/Cargo.toml | 1 + 2 files changed, 447 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fb52fb5..35381968 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,8 +39,8 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2", - "http", + "h2 0.3.26", + "http 0.2.12", "httparse", "httpdate", "itoa", @@ -76,7 +76,7 @@ checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" dependencies = [ "bytestring", "cfg-if", - "http", + "http 0.2.12", "regex", "regex-lite", "serde", @@ -312,6 +312,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.4.0" @@ -633,6 +639,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -888,6 +904,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fastrand" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -910,6 +932,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1083,7 +1120,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", "indexmap", "slab", "tokio", @@ -1144,6 +1200,40 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.9.5" @@ -1156,6 +1246,78 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.6", + "http 1.1.0", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "hyphenation" version = "0.8.4" @@ -1387,6 +1549,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ipnet" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" + [[package]] name = "is-terminal" version = "0.4.13" @@ -1642,6 +1810,23 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "neli" version = "0.6.4" @@ -1782,6 +1967,50 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2004,7 +2233,7 @@ dependencies = [ "flate2", "hex", "procfs-core", - "rustix 0.38.38", + "rustix 0.38.39", ] [[package]] @@ -2141,6 +2370,49 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.12.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.6", + "http 1.1.0", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + [[package]] name = "ring" version = "0.17.8" @@ -2226,9 +2498,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -2309,6 +2581,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2326,6 +2607,29 @@ dependencies = [ "sha2", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.23" @@ -2549,6 +2853,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.1" @@ -2637,6 +2950,7 @@ dependencies = [ "log", "once_cell", "rand", + "reqwest", "rsa", "rustls", "rustls-pemfile", @@ -2648,6 +2962,40 @@ dependencies = [ "uuid", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix 0.38.39", + "windows-sys 0.59.0", +] + [[package]] name = "term" version = "0.7.0" @@ -2772,6 +3120,16 @@ dependencies = [ "syn 2.0.85", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" @@ -2796,6 +3154,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.40" @@ -2816,6 +3180,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.17.0" @@ -2896,6 +3266,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2912,6 +3288,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2944,6 +3329,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.95" @@ -2973,6 +3370,16 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +[[package]] +name = "web-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.4.2" @@ -2982,7 +3389,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.38", + "rustix 0.38.39", ] [[package]] @@ -3043,7 +3450,7 @@ checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" dependencies = [ "windows-implement", "windows-interface", - "windows-result", + "windows-result 0.1.2", "windows-targets 0.52.6", ] @@ -3069,6 +3476,17 @@ dependencies = [ "syn 2.0.85", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result 0.2.0", + "windows-strings", + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.1.2" @@ -3078,6 +3496,25 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/sysminion/Cargo.toml b/sysminion/Cargo.toml index 1bdd18c0..c8dee472 100644 --- a/sysminion/Cargo.toml +++ b/sysminion/Cargo.toml @@ -31,3 +31,4 @@ once_cell = "1.20.2" sysinfo = { version = "0.32.0", features = ["linux-tmpfs"] } rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } uuid = { version = "1.11.0", features = ["v4"] } +reqwest = "0.12.9" From 905348bc0263f330d2df033d4cdef14ad32e0991 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 22:09:21 +0100 Subject: [PATCH 085/148] Remove dead code --- sysminion/src/proto.rs | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index 2f41571e..d534a102 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -1,10 +1,6 @@ pub mod msg { - use crate::{ - minion::{request, MINION_SID}, - traits, - }; + use crate::{minion::MINION_SID, traits}; use libsysinspect::{ - cfg::mmconf::MinionConfig, proto::{rqtypes::RequestType, MinionMessage}, util::dataconv, }; @@ -12,8 +8,6 @@ pub mod msg { proto::{MasterMessage, ProtoConversion}, SysinspectError, }; - use std::sync::Arc; - use tokio::{net::tcp::OwnedWriteHalf, sync::Mutex}; /// Make pong message pub fn get_pong() -> Vec { @@ -29,30 +23,6 @@ pub mod msg { vec![] } - /// Send ehlo - pub async fn send_ehlo(stream: Arc>, cfg: MinionConfig) -> Result<(), SysinspectError> { - let r = MinionMessage::new( - dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), - RequestType::Ehlo, - MINION_SID.to_string(), - ); - - log::info!("Ehlo on {}", cfg.master()); - request(stream, r.sendable()?).await; - Ok(()) - } - - pub async fn send_registration( - stream: Arc>, cfg: MinionConfig, pbk_pem: String, - ) -> Result<(), SysinspectError> { - let r = - MinionMessage::new(dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); - - log::info!("Registration request to {}", cfg.master()); - request(stream, r.sendable()?).await; - Ok(()) - } - /// Get message pub fn payload_to_msg(data: Vec) -> Result { let data = match String::from_utf8(data) { From 65eebe768c01ce582a90076bc89744cbf83e9fa3 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 22:09:53 +0100 Subject: [PATCH 086/148] Refactor minion as class --- sysminion/src/main.rs | 2 +- sysminion/src/minion.rs | 303 +++++++++++++++++++++++++--------------- 2 files changed, 193 insertions(+), 112 deletions(-) diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 07249e94..2c44420d 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -49,7 +49,7 @@ async fn main() -> std::io::Result<()> { let fp = params.get_one::("register").cloned(); if *params.get_one::("start").unwrap_or(&false) || fp.is_some() { let cfp = params.get_one::("config"); - if let Err(err) = minion::minion(PathBuf::from(cfp.map_or("", |v| v)), fp).await { + if let Err(err) = minion::minion(cfp.map_or("", |v| v), fp).await { log::error!("Unable to start minion: {}", err); return Ok(()); } diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index d9f55af2..43fead21 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,157 +1,238 @@ -use crate::proto; +use crate::{proto, rsa::MinionRSAKeyManager, traits}; use libsysinspect::{ cfg::{self, mmconf::MinionConfig}, - proto::{errcodes::ProtoErrorCode, rqtypes::RequestType}, - rsa, SysinspectError, + proto::{errcodes::ProtoErrorCode, rqtypes::RequestType, MinionMessage, ProtoConversion}, + rsa, + util::dataconv, + SysinspectError, }; use once_cell::sync::Lazy; use std::{path::PathBuf, sync::Arc}; -use tokio::net::TcpStream; +use tokio::net::{tcp::OwnedReadHalf, TcpStream}; use tokio::sync::Mutex; +use tokio::{io::AsyncReadExt, sync::mpsc}; use tokio::{io::AsyncWriteExt, net::tcp::OwnedWriteHalf}; -use tokio::{ - io::{AsyncReadExt, BufReader}, - sync::mpsc, -}; use uuid::Uuid; /// Session Id of the minion pub static MINION_SID: Lazy = Lazy::new(|| Uuid::new_v4().to_string()); -/// Talk-back to the master -pub async fn request(stream: Arc>, msg: Vec) { - let mut stm = stream.lock().await; +pub struct SysMinion { + cfg: MinionConfig, + fingerprint: Option, + kman: MinionRSAKeyManager, - if let Err(e) = stm.write_all(&(msg.len() as u32).to_be_bytes()).await { - log::error!("Failed to send message length to master: {}", e); - return; - } + rstm: Arc>, + wstm: Arc>, +} - if let Err(e) = stm.write_all(&msg).await { - log::error!("Failed to send message to master: {}", e); - return; +impl SysMinion { + pub async fn new(cfp: &str, fingerprint: Option) -> Result, SysinspectError> { + let mut cfp = PathBuf::from(cfp); + if !cfp.exists() { + cfp = cfg::select_config(None)?; + } + + let cfg = MinionConfig::new(cfp)?; + let (rstm, wstm) = TcpStream::connect(cfg.master()).await.unwrap().into_split(); + Ok(Arc::new(SysMinion { + cfg, + fingerprint, + kman: MinionRSAKeyManager::new(None)?, + rstm: Arc::new(Mutex::new(rstm)), + wstm: Arc::new(Mutex::new(wstm)), + })) } - if let Err(e) = stm.flush().await { - log::error!("Failed to flush writer to master: {}", e); - } else { - log::trace!("To master: {}", String::from_utf8_lossy(&msg)); + pub fn as_ptr(self: &Arc) -> Arc { + Arc::clone(&self) } -} -/// Minion routine -pub async fn minion(mut cfp: PathBuf, fingerprint: Option) -> Result<(), SysinspectError> { - let fingerprint = fingerprint.unwrap_or_default(); - if !cfp.exists() { - cfp = cfg::select_config(None)?; + /// Talk-back to the master + async fn request(&self, msg: Vec) { + let mut stm = self.wstm.lock().await; + + if let Err(e) = stm.write_all(&(msg.len() as u32).to_be_bytes()).await { + log::error!("Failed to send message length to master: {}", e); + return; + } + + if let Err(e) = stm.write_all(&msg).await { + log::error!("Failed to send message to master: {}", e); + return; + } + + if let Err(e) = stm.flush().await { + log::error!("Failed to flush writer to master: {}", e); + } else { + log::trace!("To master: {}", String::from_utf8_lossy(&msg)); + } } - let cfg = MinionConfig::new(cfp)?; - //let st = traits::get_traits(); - let mkeys = crate::rsa::MinionRSAKeyManager::new(None)?; // XXX: Get optional root from the configuration - - let (rstm, wstm) = TcpStream::connect(cfg.master()).await?.into_split(); - let wstm: Arc> = Arc::new(Mutex::new(wstm)); - let (_w_chan, mut r_chan) = mpsc::channel(100); - - // Data exchange - let wtsm_c = wstm.clone(); - tokio::spawn(async move { - let mut input = BufReader::new(rstm); - - loop { - let mut buff = [0u8; 4]; - if let Err(e) = input.read_exact(&mut buff).await { - log::trace!("Unknown message length from the master: {}", e); - break; - } - let msg_len = u32::from_be_bytes(buff) as usize; - let mut msg = vec![0u8; msg_len]; - if let Err(e) = input.read_exact(&mut msg).await { - log::error!("Invalid message from the master: {}", e); - break; + pub async fn do_qmsg(self: Arc) { + let (_w_chan, mut r_chan) = mpsc::channel(100); + let cls = Arc::new(self.clone()); + tokio::spawn(async move { + while let Some(msg) = r_chan.recv().await { + cls.request(msg).await; } + }); + } - let msg = match proto::msg::payload_to_msg(msg) { - Ok(msg) => msg, - Err(err) => { - log::error!("Error getting network payload as message: {err}"); - continue; + pub async fn do_proto(self: Arc) -> Result<(), SysinspectError> { + let rstm = Arc::clone(&self.rstm); + let cls = Arc::new(self.clone()); + + tokio::spawn(async move { + //let mut input = BufReader::new(rstm.lock().await); + loop { + let mut buff = [0u8; 4]; + if let Err(e) = rstm.lock().await.read_exact(&mut buff).await { + log::trace!("Unknown message length from the master: {}", e); + break; } - }; - - log::trace!("Received: {:?}", msg); + let msg_len = u32::from_be_bytes(buff) as usize; - match msg.req_type() { - RequestType::Add => { - log::debug!("Master accepts registration"); + let mut msg = vec![0u8; msg_len]; + if let Err(e) = rstm.lock().await.read_exact(&mut msg).await { + log::error!("Invalid message from the master: {}", e); + break; } - RequestType::Reconnect => { - log::debug!("Master requires reconnection"); - log::info!("{}", msg.payload()); - std::process::exit(0); - } + let msg = match proto::msg::payload_to_msg(msg) { + Ok(msg) => msg, + Err(err) => { + log::error!("Error getting network payload as message: {err}"); + continue; + } + }; - RequestType::Remove => { - log::debug!("Master asks to unregister"); - } - RequestType::Command => { - log::debug!("Master sends a command"); - match msg.get_retcode() { - ProtoErrorCode::AlreadyConnected => { - if MINION_SID.eq(msg.payload()) { - log::error!("Another minion from this machine is already connected"); - std::process::exit(1); + log::trace!("Received: {:?}", msg); + + match msg.req_type() { + RequestType::Add => { + log::debug!("Master accepts registration"); + } + + RequestType::Reconnect => { + log::debug!("Master requires reconnection"); + log::info!("{}", msg.payload()); + std::process::exit(0); + } + + RequestType::Remove => { + log::debug!("Master asks to unregister"); + } + RequestType::Command => { + log::debug!("Master sends a command"); + match msg.get_retcode() { + ProtoErrorCode::AlreadyConnected => { + if MINION_SID.eq(msg.payload()) { + log::error!("Another minion from this machine is already connected"); + std::process::exit(1); + } + } + ret => { + log::debug!("Return code {:?} not yet implemented", ret); } - } - ret => { - log::debug!("Return code {:?} not yet implemented", ret); } } + RequestType::Traits => { + log::debug!("Master requests traits"); + } + RequestType::AgentUnknown => { + let pbk_pem = msg.payload(); // Expected PEM RSA pub key + let (_, pbk) = rsa::keys::from_pem(None, Some(pbk_pem)).unwrap(); + let fpt = rsa::keys::get_fingerprint(&pbk.unwrap()).unwrap(); + + log::error!("Minion is not registered"); + log::info!("Master fingerprint: {}", fpt); + std::process::exit(1); + } + RequestType::Ping => { + cls.request(proto::msg::get_pong()).await; + } + _ => { + log::error!("Unknown request type"); + } } - RequestType::Traits => { - log::debug!("Master requests traits"); - } - RequestType::AgentUnknown => { - let pbk_pem = msg.payload(); // Expected PEM RSA pub key - let (_, pbk) = rsa::keys::from_pem(None, Some(pbk_pem)).unwrap(); - let fpt = rsa::keys::get_fingerprint(&pbk.unwrap()).unwrap(); - - log::error!("Minion is not registered"); - log::info!("Master fingerprint: {}", fpt); - std::process::exit(1); - } - RequestType::Ping => { - request(wtsm_c.clone(), proto::msg::get_pong()).await; - } - _ => { - log::error!("Unknown request type"); - } + + //request(wtsm_c.clone(), response).await; } + }); + Ok(()) + } - //request(wtsm_c.clone(), response).await; - } - }); + /// Send ehlo + pub async fn send_ehlo(self: Arc) -> Result<(), SysinspectError> { + let r = MinionMessage::new( + dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), + RequestType::Ehlo, + MINION_SID.to_string(), + ); + + log::info!("Ehlo on {}", self.cfg.master()); + self.request(r.sendable()?).await; + Ok(()) + } - // Task to handle queued messages for the master - let qmsg_stm = wstm.to_owned(); - tokio::spawn(async move { - while let Some(msg) = r_chan.recv().await { - request(qmsg_stm.clone(), msg).await; + /// Send registration request + pub async fn send_registration(self: Arc, pbk_pem: String) -> Result<(), SysinspectError> { + let r = + MinionMessage::new(dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); + + log::info!("Registration request to {}", self.cfg.master()); + self.request(r.sendable()?).await; + Ok(()) + } + + /// Download a file from master + async fn download_file(self: Arc, url: String, fname: String) { + async fn fetch_file(url: &str, filename: &str) -> Result { + let url = format!("{}/{}", url.strip_suffix("/").unwrap_or_default(), filename); + let rsp = match reqwest::get(&url).await { + Ok(rsp) => rsp, + Err(err) => { + return Err(SysinspectError::MinionGeneralError(format!("{}", err))); + } + }; + + Ok(match rsp.status() { + reqwest::StatusCode::OK => match rsp.text().await { + Ok(data) => data, + Err(err) => { + return Err(SysinspectError::MinionGeneralError(format!("{}", err))); + } + }, + reqwest::StatusCode::NOT_FOUND => return Err(SysinspectError::MinionGeneralError(format!("File not found"))), + _ => return Err(SysinspectError::MinionGeneralError(format!("Unknown status"))), + }) } - }); + tokio::spawn(async move { + match fetch_file(&url, &fname).await { + Ok(data) => log::debug!("Result returned as {:#?}", data), + Err(err) => log::error!("{err}"), + } + }); + } +} + +pub async fn minion(cfp: &str, fingerprint: Option) -> Result<(), SysinspectError> { + let minion = SysMinion::new(cfp, fingerprint).await?; + minion.as_ptr().do_proto().await?; + // minion.as_ptr().download_file("http://127.0.0.1:4201".to_string(), "models/inherited/model.cfg".to_string()).await; // Messages - if !fingerprint.is_empty() { - proto::msg::send_registration(wstm.clone(), cfg, mkeys.get_pubkey_pem()).await?; + if minion.fingerprint.is_some() { + minion.as_ptr().send_registration(minion.kman.get_pubkey_pem()).await?; } else { // ehlo - proto::msg::send_ehlo(wstm.clone(), cfg).await?; + minion.as_ptr().send_ehlo().await?; } // Keep the client alive until Ctrl+C is pressed tokio::signal::ctrl_c().await.expect("Failed to listen for ctrl_c"); log::info!("Shutting down client."); + Ok(()) } From 494426feb488ba006dfbb754cf84923bd59ecc5e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 22:12:38 +0100 Subject: [PATCH 087/148] Remove unused import --- sysminion/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 2c44420d..31cb0c8e 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -7,7 +7,7 @@ mod traits; use clidef::cli; use libsysinspect::logger; use log::LevelFilter; -use std::{env, path::PathBuf}; +use std::env; static APPNAME: &str = "sysminion"; static VERSION: &str = "0.0.1"; From 6aeda494e22949b11ac9a6ffdb6027c76d3bcb5c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 22:13:00 +0100 Subject: [PATCH 088/148] Remove redundant refcount --- sysminion/src/minion.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 43fead21..6c7709df 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -45,7 +45,7 @@ impl SysMinion { } pub fn as_ptr(self: &Arc) -> Arc { - Arc::clone(&self) + Arc::clone(self) } /// Talk-back to the master From c6139287beebe31dcc5a2a38540c62e0c6ae9f22 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 9 Nov 2024 22:13:17 +0100 Subject: [PATCH 089/148] Remove unnecessary macros --- sysminion/src/minion.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 6c7709df..a998541b 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -204,8 +204,8 @@ impl SysMinion { return Err(SysinspectError::MinionGeneralError(format!("{}", err))); } }, - reqwest::StatusCode::NOT_FOUND => return Err(SysinspectError::MinionGeneralError(format!("File not found"))), - _ => return Err(SysinspectError::MinionGeneralError(format!("Unknown status"))), + reqwest::StatusCode::NOT_FOUND => return Err(SysinspectError::MinionGeneralError("File not found".to_string())), + _ => return Err(SysinspectError::MinionGeneralError("Unknown status".to_string())), }) } tokio::spawn(async move { From 7f953ee165f742d87b82afaaf038cd6e2cd78665 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 10 Nov 2024 02:26:57 +0100 Subject: [PATCH 090/148] Fix file downloader --- libsysinspect/src/cfg/mmconf.rs | 12 ++++++++++-- sysminion/src/minion.rs | 12 +++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/libsysinspect/src/cfg/mmconf.rs b/libsysinspect/src/cfg/mmconf.rs index b748cdf2..ed264df1 100644 --- a/libsysinspect/src/cfg/mmconf.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -14,7 +14,10 @@ pub struct MinionConfig { master_ip: String, #[serde(rename = "master.port")] - master_port: u32, + master_port: Option, + + #[serde(rename = "master.fileserver.port")] + master_fileserver_port: Option, } impl MinionConfig { @@ -33,7 +36,12 @@ impl MinionConfig { /// Return master addr pub fn master(&self) -> String { - format!("{}:{}", self.master_ip, self.master_port) + format!("{}:{}", self.master_ip, self.master_port.unwrap_or(DEFAULT_PORT)) + } + + /// Return master fileserver addr + pub fn fileserver(&self) -> String { + format!("{}:{}", self.master_ip, self.master_fileserver_port.unwrap_or(DEFAULT_FILESERVER_PORT)) } } diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index a998541b..fdd5a935 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -187,10 +187,9 @@ impl SysMinion { } /// Download a file from master - async fn download_file(self: Arc, url: String, fname: String) { + async fn download_file(self: Arc, fname: String) { async fn fetch_file(url: &str, filename: &str) -> Result { - let url = format!("{}/{}", url.strip_suffix("/").unwrap_or_default(), filename); - let rsp = match reqwest::get(&url).await { + let rsp = match reqwest::get(format!("http://{}/{}", url, filename)).await { Ok(rsp) => rsp, Err(err) => { return Err(SysinspectError::MinionGeneralError(format!("{}", err))); @@ -208,8 +207,9 @@ impl SysMinion { _ => return Err(SysinspectError::MinionGeneralError("Unknown status".to_string())), }) } + let addr = self.cfg.fileserver(); tokio::spawn(async move { - match fetch_file(&url, &fname).await { + match fetch_file(&addr, &fname).await { Ok(data) => log::debug!("Result returned as {:#?}", data), Err(err) => log::error!("{err}"), } @@ -220,7 +220,9 @@ impl SysMinion { pub async fn minion(cfp: &str, fingerprint: Option) -> Result<(), SysinspectError> { let minion = SysMinion::new(cfp, fingerprint).await?; minion.as_ptr().do_proto().await?; - // minion.as_ptr().download_file("http://127.0.0.1:4201".to_string(), "models/inherited/model.cfg".to_string()).await; + + // Example downloading file + //minion.as_ptr().download_file("models/inherited/model.cfg".to_string()).await; // Messages if minion.fingerprint.is_some() { From 9d65ba8c45c9c0c2f7ea24334ff56bf7eaddec74 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 10 Nov 2024 20:29:17 +0100 Subject: [PATCH 091/148] Describe distributed model principles --- docs/genusage/distributed_model.rst | 156 ++++++++++++++++++++++++++++ docs/genusage/overview.rst | 27 +++++ 2 files changed, 183 insertions(+) create mode 100644 docs/genusage/distributed_model.rst create mode 100644 docs/genusage/overview.rst diff --git a/docs/genusage/distributed_model.rst b/docs/genusage/distributed_model.rst new file mode 100644 index 00000000..a19b9bfb --- /dev/null +++ b/docs/genusage/distributed_model.rst @@ -0,0 +1,156 @@ +.. raw:: html + + + +.. role:: u + :class: underlined + +.. role:: bi + :class: bolditalic + +Distributed Model +================= + +Distributed model means that the entities of it might be scattered +across the system in different "boxes" or consist of different components, +those are found in different network-connected places. + +Each entity keeps some claims about it. For example, a network consists of +IP addresses, MAC addresses, some ports, some protocols etc. Each of these +elements are *"building bricks"* of the whole network entity. Suppose, +we need to ensure that the network across many "boxes" contains specific +ports, protocols, addresses, routes etc. And each "box" has different configuration +for it. This means each part of the network will have a different set of claims. + +For example, a router, which is :bi:`a part of the network entity`, has specific +requirements on its own inside of it, and they are claims. But they are not a +:bi:`global` claims, they are only claims that are specific to that router. +However, we cannot admit that the network is healthy if only router alone works. +We want to check many other places and settings all over the related "boxes" +and see if each box on its own has their claims admitted as actual proven facts. +Only if :bi:`all pieces comes together`, only if all checks on all boxes/places/segments +are verified and are "green", only then we can admit that the "Network Entity" +works as expected. + +To achieve this, we need to distribute the model. + +.. important:: + + The model is actually always distributed. It is just the way how do you write it. + However, the model is aimed to cover the entire :bi:`system`, which is not just + one or two "boxes" or software components, processes etc. + +Model Distribution Principle +---------------------------- + +The static model is always copied to each minion "as is". But that means that each +minion supposed to be exactly the same in order to return a success result. However, +in a complex system, some "boxes" are Database, some of them are routers, some of them +are just storage systems, some of them are running middleware/services etc. And thus +all of these "boxes" render a final heterogeneous distributed system. + +How Sysinspect dissects to the targets only what's needed? + +Let's take a look at the entity description: + +.. code-block:: yaml + + # General section for all entities + entities: + router: + claims: + $: + - foo: ... + + powered: + - foobar: ... + +In this case we have an entity, callsed **"router"**, which has some default common claims, +accessible by globbing ``$`` (e.g. some admin interfaces are always there, config of hypervisor +partitions etc), and it also has claims that are valid only when it is in ``powered`` +mode. + +.. pull-quote:: + + Now, what if we have :bi:`two different routers`, and yet they have to be + together, in order to let the whole network work as expected? How to split + facts between them? + +For this, an entity needs to match a minion by its traits, and claims are defined under +the section, where traits are filtering (or selecting) :bi:`the relevant part` of the claims +description in a declarative way. + +Splitting Claims +---------------- + +Let's continue with the same router example, but add another one. Say, the other one +is running on ARM-64 architecture instead of on Intel x86_64, has much less memory +and has a different hostname. By these traits we can distinguish those two routers, +splitting claims among them. + +Here is an example: + +.. code-block:: yaml + + # General section for all entities + entities: + router: + # Router 1 + my-big-one: + traits: + system.cpu.arch: x86_64 + system.mem.total: 16GB + net.hostname: crocodile.local + system.os.vendor: f5 + + # Specific flaims only for big, powerful "f5" Intel-based router + claims: + ... + + # Router 2 + my-small-one: + traits: + system.cpu.arch: ARM64 + system.mem.total: 4GB + net.hostname: frog.local + + # Specific flaims only for ARM-based router + claims: + ... + + # Common claims for both routers + claims: + $: + - foo: ... + + powered: + - foobar: ... + +The above is the entity description that is in the master Model. However, +each minion will not get the entire model, but only :bi:`a subset` of the Model, +which is relevant to only that specific minion. + +.. note:: + + Each minion will get only :bi:`a subset` of the Model, relevant only to + the current minion traits or other attributes! + +The mechanism works very similar to the Model Inheritance: matching section +will replace the default claims section from the section that matches the minion. +In case both sections are matching a minion, then they will be merged. If they +overlap, then first wins. Therefore it is very important to be careful to point out +the difference in traits or other attributes, ensuring the model is not overlapping +on the minion side or renders wrong. + +.. important:: + + It is important to use granular and detailed targeting, in order to avoid + claims overlap between the minions, rendering false results. \ No newline at end of file diff --git a/docs/genusage/overview.rst b/docs/genusage/overview.rst new file mode 100644 index 00000000..f64b3159 --- /dev/null +++ b/docs/genusage/overview.rst @@ -0,0 +1,27 @@ +Using Sysinspect +================ + +.. note:: + + This section explains how to use Sysinspect in "solo" mode and + in the network. + + +Sysinspect can be used in two modes: + +1. Solo mode, where the entire model is for only one "box" or hardware +2. Network-connected cluster, where entities can consist from more than one element + and Sysinspect needs to gather information from different places in order to construct + a final answer about a specific entity. Such entity, for example, can be the entire + network itself. + + +To better understand how to use Sysinspect in those situation, read through the following +sections: + +.. toctree:: + :maxdepth: 2 + + distributed_model + systrails + targeting From 1f4798e463389d42edd5bf26a7af4215212d73cf Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 11 Nov 2024 13:47:55 +0100 Subject: [PATCH 092/148] Moved traits subsystem to the libsysinspect --- Cargo.lock | 62 +++++++++++++++++++ Cargo.toml | 2 +- libsysinspect/Cargo.toml | 3 + libsysinspect/src/lib.rs | 1 + .../src/traits/mod.rs | 0 .../src/traits/systraits.rs | 3 +- sysminion/src/main.rs | 1 - sysminion/src/minion.rs | 4 +- sysminion/src/proto.rs | 3 +- 9 files changed, 72 insertions(+), 7 deletions(-) rename {sysminion => libsysinspect}/src/traits/mod.rs (100%) rename {sysminion => libsysinspect}/src/traits/systraits.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 35381968..1393d150 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1678,6 +1678,7 @@ dependencies = [ "lazy_static", "log", "nix", + "once_cell", "pem", "prettytable-rs", "rand", @@ -1687,7 +1688,9 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", + "sysinfo 0.32.0", "textwrap", + "tokio", "unicode-segmentation", "walkdir", ] @@ -2100,6 +2103,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.85", +] + +[[package]] +name = "pest_meta" +version = "2.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pin-project-lite" version = "0.2.15" @@ -2247,6 +2295,14 @@ dependencies = [ "hex", ] +[[package]] +name = "qparse" +version = "0.1.0" +dependencies = [ + "pest", + "pest_derive", +] + [[package]] name = "quote" version = "1.0.37" @@ -3192,6 +3248,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unicode-ident" version = "1.0.13" diff --git a/Cargo.toml b/Cargo.toml index bf2c4787..fc42934b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ sysinfo = { version = "0.31.4", features = ["linux-tmpfs"] } [workspace] resolver = "2" -members = ["modules/sys/*", "libsysinspect", "sysmaster", "sysminion"] +members = ["modules/sys/*", "libsysinspect", "sysmaster", "sysminion", "qparse"] [profile.release] strip = true diff --git a/libsysinspect/Cargo.toml b/libsysinspect/Cargo.toml index e258d6dd..6e395a8f 100644 --- a/libsysinspect/Cargo.toml +++ b/libsysinspect/Cargo.toml @@ -11,6 +11,7 @@ indexmap = "2.6.0" lazy_static = "1.5.0" log = "0.4.22" nix = { version = "0.29.0", features = ["net", "user"] } +once_cell = "1.20.2" pem = "3.0.4" prettytable-rs = "0.10.0" rand = "0.8.5" @@ -20,6 +21,8 @@ serde = { version = "1.0.210", features = ["derive"] } serde_json = "1.0.128" serde_yaml = "0.9.34" sha2 = "0.10.8" +sysinfo = { version = "0.32.0", features = ["linux-tmpfs"] } textwrap = { version = "0.16.1", features = ["hyphenation", "terminal_size"] } +tokio = { version = "1.41.1", features = ["full"] } unicode-segmentation = "1.12.0" walkdir = "2.5.0" diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 4cb92d6e..6f03613a 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -16,6 +16,7 @@ pub mod modlib; pub mod proto; pub mod reactor; pub mod rsa; +pub mod traits; pub mod util; #[derive(Debug)] diff --git a/sysminion/src/traits/mod.rs b/libsysinspect/src/traits/mod.rs similarity index 100% rename from sysminion/src/traits/mod.rs rename to libsysinspect/src/traits/mod.rs diff --git a/sysminion/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs similarity index 99% rename from sysminion/src/traits/systraits.rs rename to libsysinspect/src/traits/systraits.rs index eb079a2b..acea2943 100644 --- a/sysminion/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -1,8 +1,7 @@ -use std::{fs, path::PathBuf}; - use indexmap::IndexMap; use once_cell::sync::Lazy; use serde_json::{json, Value}; +use std::{fs, path::PathBuf}; use tokio::sync::Mutex; use crate::traits::{ diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 31cb0c8e..0a12291e 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -2,7 +2,6 @@ mod clidef; mod minion; mod proto; mod rsa; -mod traits; use clidef::cli; use libsysinspect::logger; diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index fdd5a935..0fed5c78 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,8 +1,8 @@ -use crate::{proto, rsa::MinionRSAKeyManager, traits}; +use crate::{proto, rsa::MinionRSAKeyManager}; use libsysinspect::{ cfg::{self, mmconf::MinionConfig}, proto::{errcodes::ProtoErrorCode, rqtypes::RequestType, MinionMessage, ProtoConversion}, - rsa, + rsa, traits, util::dataconv, SysinspectError, }; diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index d534a102..acccc1e5 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -1,7 +1,8 @@ pub mod msg { - use crate::{minion::MINION_SID, traits}; + use crate::minion::MINION_SID; use libsysinspect::{ proto::{rqtypes::RequestType, MinionMessage}, + traits, util::dataconv, }; use libsysinspect::{ From edb9c28f2bc00cb90a3c0a2ba1ef3511b3db329a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 11 Nov 2024 13:56:52 +0100 Subject: [PATCH 093/148] Add query grammar --- libsysinspect/src/traits/traits_query.pest | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 libsysinspect/src/traits/traits_query.pest diff --git a/libsysinspect/src/traits/traits_query.pest b/libsysinspect/src/traits/traits_query.pest new file mode 100644 index 00000000..ca11d075 --- /dev/null +++ b/libsysinspect/src/traits/traits_query.pest @@ -0,0 +1,17 @@ +// src/query.pest + +// Define silent whitespace rule to skip over spaces and tabs +WHITESPACE = _{ " " | "\t" } + +// Define a term as one or more alphanumeric characters +term = @{ ASCII_ALPHANUMERIC+ } + +// Define "and" and "or" operators as silent rules +and_op = _{ "and" } +or_op = _{ "or" } + +// Define a group: one or more terms connected by "and" +group = { term ~ (and_op ~ term)* } + +// Define an expression: one or more groups connected by "or" +expression = { group ~ (or_op ~ group)* } From ef363305455fcd837f963aa4d527a449175b5998 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 11 Nov 2024 13:57:21 +0100 Subject: [PATCH 094/148] Update dependencies --- Cargo.lock | 10 ++-------- Cargo.toml | 2 +- libsysinspect/Cargo.toml | 2 ++ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1393d150..3e258ae6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1680,6 +1680,8 @@ dependencies = [ "nix", "once_cell", "pem", + "pest", + "pest_derive", "prettytable-rs", "rand", "regex", @@ -2295,14 +2297,6 @@ dependencies = [ "hex", ] -[[package]] -name = "qparse" -version = "0.1.0" -dependencies = [ - "pest", - "pest_derive", -] - [[package]] name = "quote" version = "1.0.37" diff --git a/Cargo.toml b/Cargo.toml index fc42934b..bf2c4787 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ sysinfo = { version = "0.31.4", features = ["linux-tmpfs"] } [workspace] resolver = "2" -members = ["modules/sys/*", "libsysinspect", "sysmaster", "sysminion", "qparse"] +members = ["modules/sys/*", "libsysinspect", "sysmaster", "sysminion"] [profile.release] strip = true diff --git a/libsysinspect/Cargo.toml b/libsysinspect/Cargo.toml index 6e395a8f..135ae67f 100644 --- a/libsysinspect/Cargo.toml +++ b/libsysinspect/Cargo.toml @@ -13,6 +13,8 @@ log = "0.4.22" nix = { version = "0.29.0", features = ["net", "user"] } once_cell = "1.20.2" pem = "3.0.4" +pest = "2.7.14" +pest_derive = "2.7.14" prettytable-rs = "0.10.0" rand = "0.8.5" regex = "1.10.6" From f16c969c28596229104be4745f7b3804ed6ce85b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 11 Nov 2024 13:57:40 +0100 Subject: [PATCH 095/148] Add query parser implementation --- libsysinspect/src/traits/mod.rs | 46 ++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/libsysinspect/src/traits/mod.rs b/libsysinspect/src/traits/mod.rs index 08a0b7ce..f120e5b9 100644 --- a/libsysinspect/src/traits/mod.rs +++ b/libsysinspect/src/traits/mod.rs @@ -1,9 +1,12 @@ mod systraits; - use once_cell::sync::Lazy; +use pest::Parser; +use pest_derive::Parser; use std::sync::Mutex; use systraits::SystemTraits; +use crate::SysinspectError; + /// Standard Traits pub static SYS_ID: &str = "system.id"; pub static SYS_OS_KERNEL: &str = "system.kernel"; @@ -38,3 +41,44 @@ pub fn get_traits() -> SystemTraits { SystemTraits::default() } + +#[derive(Parser)] +#[grammar = "traits/traits_query.pest"] +struct QueryParser; + +/// Parse a very simple traits query. It returns an array of OR arrays, containing AND values. +/// Example: +/// +/// "foo and bar or baz" +/// +/// This yields to the following structure: +/// +/// [[foo, bar], [baz]] +/// +/// Each inner array should be treated with AND operator. +pub fn get_traits_query(input: &str) -> Result>, SysinspectError> { + let mut out = Vec::new(); + let mut pairs = match QueryParser::parse(Rule::expression, input) { + Ok(prs) => prs, + Err(err) => return Err(SysinspectError::ModelDSLError(format!("Invalid query: {err}"))), + }; + + let expr = match pairs.next() { + Some(expr) => expr, + None => return Ok(out), + }; + + for grp in expr.into_inner() { + if grp.as_rule() == Rule::group { + let mut terms = Vec::new(); + for t_pair in grp.into_inner() { + if t_pair.as_rule() == Rule::term { + terms.push(t_pair.as_str().to_string()); + } + } + out.push(terms); + } + } + + Ok(out) +} From e4147853994abaee4a6a0d2889969a08b78d2a14 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 11 Nov 2024 14:36:44 +0100 Subject: [PATCH 096/148] Update documentation on usage and targeting minions --- docs/genusage/overview.rst | 2 +- docs/genusage/systraits.rst | 68 ++++++++++++++++ docs/genusage/targeting.rst | 154 ++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 4 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 docs/genusage/systraits.rst create mode 100644 docs/genusage/targeting.rst diff --git a/docs/genusage/overview.rst b/docs/genusage/overview.rst index f64b3159..a6d985e5 100644 --- a/docs/genusage/overview.rst +++ b/docs/genusage/overview.rst @@ -23,5 +23,5 @@ sections: :maxdepth: 2 distributed_model - systrails + systraits targeting diff --git a/docs/genusage/systraits.rst b/docs/genusage/systraits.rst new file mode 100644 index 00000000..ccf9061f --- /dev/null +++ b/docs/genusage/systraits.rst @@ -0,0 +1,68 @@ +.. raw:: html + + + +.. role:: u + :class: underlined + +.. role:: bi + :class: bolditalic + +.. _systraits: + +System Traits +============= + +.. note:: + + Definition and description of system traits and their purpose. + +Traits are essentially static attributes of a minion. They can be a literally anything +in a form of key/value. There are different kinds of traits: + +**Common** + + Common traits are given to each minion automatically. They are typical system + information and anything else that can be commonly fetched out of the "box": OS info, + kernel version, memory size, network settings, hostnames, machine Id etc. + +**Custom** + + Custom traits are static data that set explicity onto a minion. Any data in + key/value form. They are usually various labels, rack number, physical floor, + Asset Tag, serial number etc. + +**Dynamic** + + Dynamic traits are custom functions, where data obtained by relevant modules. + essentially, they are just like normal modules, except the resulting data is stored as + a criterion by which a specific minion is targeted. For example, *"memory less than X"*, + or *"runs process Y"* etc. + +Listing Traits +-------------- + +(TBD) How to see traits of minions. + +Using Traits in a Model +----------------------- + +(TBD) Targeting distributed entities by traits etc. + +Static Minion Traits +-------------------- + +(TBD) As a part of minion configuration. + +Populating Traits +----------------- + +(TBD) How to push traits to all minions and fetch theirs \ No newline at end of file diff --git a/docs/genusage/targeting.rst b/docs/genusage/targeting.rst new file mode 100644 index 00000000..13493d16 --- /dev/null +++ b/docs/genusage/targeting.rst @@ -0,0 +1,154 @@ +.. raw:: html + + + +.. role:: u + :class: underlined + +.. role:: bi + :class: bolditalic + +Targeting Entities +================== + +.. note:: + + Entities are bound to the specific hardware, which is related to a specific minion. + This document explains how to target specific minions to complete entity description. + +General +------- + +Sysinspect has a query mechanism where master can target remote minions by a specific +criteria. Similarly, the Model itself can be called from different "entry points". + +**Checkbook** + + A Checkbook is basically a list of "entry points" to a complex trees of entities, + essentially a group of entities that form a feature or a set of features to be checked. + +**Entities** + + A regular Model entities. This type of entry is usually used to narrow down assessment + path. + + +Checkbook query is using path-like tuples to target a feature (a group of entities etc) +in the following format: + +.. code-block:: text + :caption: Query synopsis + + "/[entity]/[state] [traits query]" + +Since there can be many models, it is essential to select one, therefore a model Id is +always required. If ``entity`` and/or ``state`` are not specified, they are defaulted to +``$`` (all). + +.. code-block:: bash + :caption: Example of Model targeting + + sysinspect "router/network/online" + +In the example above, a network is verified in a router only when it supposed to be online. +Under the hood, omitted parts are translated to "all" (``$``). E.g. ``router/network`` is +translated as ``router/network/$``, or Model name alone e.g. ``router`` is translated to +``router/$/$`` etc. + +Traits query is separated with a space and essentially a list of possible traits with their +values and logical operations. See :ref:`query_targeting` for more details. + +Using Traits +------------ + +Every minion, running on the system can be targeted with specific criterion, describing it. +Each :bi:`minion` has a set of attributes, called :ref:`systraits`. They are used to identify +and target minions directly or from the Model. + +.. warning:: + + Using dynamic or static traits strongly depends on the use case of the Model. In terms of + portability, even though static traits are "hard-coding" claims, they are stable to the + system architecture. Likewise dynamic traits are move flexible, but they can also be more + difficult to debug, when they clash with each other. + + +.. _query_targeting: + +Query Targeting +--------------- + +Additionally, traits can be incorporated in the query. The main use of traits are +within the model, but sometimes one needs to target only a specific entity that has scope +exclusively bound to a specific minion. In the nutshell, the idea is to filter-out other +irrelevant minions, carrying *similar* entities. + +Synopsis of the query is as following: + +.. code-block:: text + :caption: Query synopsis + + ... + +Query does not support grouping with `( ... )` parentheses and is read from left to right. +Example: + +.. code-block:: bash + + "system.os.vendor:Debian and system.os.arch:ARM64 + or system.os.vendor:RHEL and system.os.arch:x86_64" + +The expression above is telling Sysinspect to target minions, those are: + +1. Running Linux Debian on ARM-64 architecture +2. Running Linux RHEL on x86_64 architecture + +As it is very clear from the example above, the use of operators must be careful. Switch +of them differently will cause different results. For example: + +.. code-block:: bash + + "system.os.vendor:Debian or system.os.arch:ARM64 + and system.os.vendor:RHEL or system.os.arch:x86_64" + +The expression above is telling Sysinspect to target minions, those are: + +1. Running Linux Debian +2. Running Linux RHEL on x86_64 architecture +3. Running on ARM-64 architecture + + +Distributed Entity +------------------ + +.. warning:: + + ⚠️ Planned feature for future releases, not implemented yet. + +Some entities can be distributed across different boxes. For example, "Backup over WiFi" +may involve a router, a WiFi antennae online and a storage with all disks in the RAID online. +However, Sysinspect can query that feature directly by its label. + +The following synopsis of the distributed entity notation in Checkbook: + +.. code-block:: text + + : + : + +For example, the use case of "Backup over WiFi" would be expressed the following way: + +.. code-block:: yaml + + backup_over_wifi: + - antennae: 'status:online and freq_ghz:5' # Use of custom traits via functions + - raid: 'system.os.vendor:Debian and net.hostname:storage.local' + - router: 'system.os.mem:16GB and &raid' # References "raid" group by label diff --git a/docs/index.rst b/docs/index.rst index 9dbf824a..4e96438f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,7 @@ Welcome to the Project Sysinspect! :maxdepth: 1 global_config + genusage/overview modeldescr/overview moddev/overview moddescr/overview From c00cbbdbee5e3c5975210ccad8772cebb26d2e9b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 11 Nov 2024 22:06:41 +0100 Subject: [PATCH 097/148] Update documentation on traits and dynamic traits use --- docs/genusage/distributed_model.rst | 4 + docs/genusage/overview.rst | 23 ++++ docs/genusage/systraits.rst | 159 ++++++++++++++++++++++++++-- docs/modeldescr/overview.rst | 2 + 4 files changed, 182 insertions(+), 6 deletions(-) diff --git a/docs/genusage/distributed_model.rst b/docs/genusage/distributed_model.rst index a19b9bfb..78903131 100644 --- a/docs/genusage/distributed_model.rst +++ b/docs/genusage/distributed_model.rst @@ -16,6 +16,8 @@ .. role:: bi :class: bolditalic +.. _distributed_model: + Distributed Model ================= @@ -88,6 +90,8 @@ For this, an entity needs to match a minion by its traits, and claims are define the section, where traits are filtering (or selecting) :bi:`the relevant part` of the claims description in a declarative way. +.. _splitting_claims: + Splitting Claims ---------------- diff --git a/docs/genusage/overview.rst b/docs/genusage/overview.rst index a6d985e5..77c2cfc4 100644 --- a/docs/genusage/overview.rst +++ b/docs/genusage/overview.rst @@ -15,6 +15,29 @@ Sysinspect can be used in two modes: a final answer about a specific entity. Such entity, for example, can be the entire network itself. +19 Seconds Tutorial +------------------- + +So you wrote a Model, using "Model Description" documentation and placed it to +``/etc/sysinspect/models`` directory on your Master machine as ``my_model``. + +Then just call the entire model across all minions: + +.. code-block:: bash + + sysinspect "my_model" + +You can call only a subset of your module, such as a specific state of a specific entity. +For example: + +.. code-block:: bash + + sysinspect "my_model/my_entity/my_state" + +For more information go ahead and dive in! + +Diving In +--------- To better understand how to use Sysinspect in those situation, read through the following sections: diff --git a/docs/genusage/systraits.rst b/docs/genusage/systraits.rst index ccf9061f..34355185 100644 --- a/docs/genusage/systraits.rst +++ b/docs/genusage/systraits.rst @@ -50,19 +50,166 @@ in a form of key/value. There are different kinds of traits: Listing Traits -------------- -(TBD) How to see traits of minions. +To list minion's traits, is enough to target a minion by its Id or hostname: + +.. code-block:: bash + + $ sysinspect --minions + ... + + $ sysinspect --info Using Traits in a Model ----------------------- -(TBD) Targeting distributed entities by traits etc. +Using traits in a model is described in :ref:`splitting_claims` chapter of the :ref:`distributed_model` document. Static Minion Traits -------------------- -(TBD) As a part of minion configuration. +Traits can be also custom static data, which is placed in a minion configuration. Traits are just +YAML files with key/value format, placed in ``$SYSINSPECT/traits`` directory of a minion. The naming +of those files is not important, they will be anyway merged into one tree. Important is to ensure +that trait keys do not repeat, so they do not overwrite each other. The ``$SYSINSPECT`` directory +is ``/etc/sysinspect`` by default or is defined in the minion configuration. + +Example of a trait file: + +.. code-block:: yaml + :caption: File: ``/etc/sysinspect/traits/example.trait`` + + traits: + name: Fred + +From now on, the minion can be targeded by the trait ``name``: + +.. code-block:: bash + :caption: Targeting a minion by a custom trait + + sysinspect "my_model/my_entity name:Fred" + +.. code-block:: + +Populating Static Traits +------------------------ + +Populating traits is done in two steps: + +1. Writing a specific static trait in a trait description +2. Populating the trait description to all targeted minions + +Synopsis of a trait description as follows: + +.. code-block:: text + :caption: Synopsis + + : + [machine-id]: + - [list] + [hostname]: + - [list] + [traits]: + [key]: [value] + : + [key]: [value] + + # Only for dynamic traits (functions) + [functions]: + - [list] + +For example, to make an alias for all Ubuntu running machines, the following valid trait description: + +.. code-block:: yaml + :caption: An alias to a system trait + + # This is to select what minions should have + # the following traits assigned + query: + traits: + - system.os.kernel.version: 6.* + + # Actual traits to be assigned + traits: + kernel: six + +Now it is possible to call all minions with any kernel of major version 6 like so: + +.. code-block:: bash + :caption: Target minions by own alias + + sysinspect "my_model/my_entity kernel:six" + +The section ``functions`` is used for the dynamic traits, described below. + +Dynamic Traits +-------------- + +Dynamic traits are functions that are doing something on the machine. Since those functions +are standalone executables, they do not accept any parameters. Functions are the same modules +like any other modules and using the same protocol with the JSON format. The difference is that +the module should return key/value structure. For example: + +.. code-block:: json + + { + "key": "value", + } + +Example of using a custom module: + +.. code-block:: bash + :caption: File: ``my_trait.sh`` + + #!/usr/bin/bash + kernel=$(uname -r) + echo $(printf '{"kernel.release": "%s"}' $kernel) + +The output of this script is a JSON key/value structure: + +.. code-block:: json + :caption: Example output + + { + "kernel.release": "5.19.0-50-generic" + } + +The function module must be portable, i.e. Minion has no responsibility to ensuring if the +function module is actionable or not on a target system. I.e. user must ensure that the target +system where the particular minion is running, should be equipped with Bash in ``/usr/bin`` +directory. + +Any modules that return non-zero return like system error more than ``1`` is simply ignored +and error is logged. + +Populating Dynamic Traits +------------------------- + +To populate dynamic trait there are three steps for this: + +1. Writing a specific trait in a Trait Description +2. Placing the trait module to the file server so the minions can download it +3. Populating the Trait Description to all targeted minions + +To write a specific trait in a Trait Description, the ``functions`` section must be specified. +Example: + +.. code-block:: yaml + + functions: + # Specify a relative path on the fileserver + - /functions/my_trait.sh + +The script ``my_trait.sh`` will be copied to ``$SYSINSPECT/functions``. When the minion starts, +it will execute each function in alphabetical oder, read the JSON output and merge the result +into the common traits tree. Then the traits tree will be synchronised with the Master. + +.. important:: -Populating Traits ------------------ + While function traits are dynamic, they are still should be treated as static data. -(TBD) How to push traits to all minions and fetch theirs \ No newline at end of file +While function sounds dynamic, the trait is still an attribute :bi:`by which` a minion is queried. +This means if the attribute will be different at every minion startup, it might be useless +to target a minion by such attribute, unless it is matching to some regular expression. There +might be a rare use cases, such as *"select minion or not, depending on its mood"* (because the +function returns every time a different value), but generally this sort of dynamism is nearly +outside of the scope of traits system. \ No newline at end of file diff --git a/docs/modeldescr/overview.rst b/docs/modeldescr/overview.rst index 9052ac2c..6b630176 100644 --- a/docs/modeldescr/overview.rst +++ b/docs/modeldescr/overview.rst @@ -4,6 +4,8 @@ Model Description .. note:: Document explains the Model Definition and its components. +.. _model_description: + The Model is essentially a configuration of a system. It is written in YAML format, and it is following a specific expression schema and logic. From fab0d4bf2497c0601665972fa6fc16a0f560297a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 14:33:26 +0100 Subject: [PATCH 098/148] Refactor configs for master/minion --- libsysinspect/src/cfg/mmconf.rs | 46 ++++++++++++++++++++++++++++++--- sysmaster/src/dataserv/fls.rs | 10 +++---- sysmaster/src/master.rs | 2 +- sysmaster/src/registry/mkb.rs | 11 ++++---- sysmaster/src/registry/mod.rs | 6 ----- sysminion/src/minion.rs | 10 +++++-- sysminion/src/rsa.rs | 16 +++++------- 7 files changed, 68 insertions(+), 33 deletions(-) diff --git a/libsysinspect/src/cfg/mmconf.rs b/libsysinspect/src/cfg/mmconf.rs index ed264df1..34591c70 100644 --- a/libsysinspect/src/cfg/mmconf.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -3,19 +3,37 @@ use serde::{Deserialize, Serialize}; use serde_yaml::{from_str, from_value, Value}; use std::{fs, path::PathBuf}; -static DEFAULT_ADDR: &str = "0.0.0.0"; -static DEFAULT_PORT: u32 = 4200; -static DEFAULT_FILESERVER_PORT: u32 = 4201; -static DEFAULT_SOCKET: &str = "/var/run/sysinspect-master.socket"; +pub static DEFAULT_ADDR: &str = "0.0.0.0"; +pub static DEFAULT_PORT: u32 = 4200; +pub static DEFAULT_FILESERVER_PORT: u32 = 4201; +pub static DEFAULT_SOCKET: &str = "/var/run/sysinspect-master.socket"; +pub static DEFAULT_SYSINSPECT_ROOT: &str = "/etc/sysinspect"; + +pub static CFG_MINION_KEYS: &str = "minion-keys"; +pub static CFG_FILESERVER_ROOT: &str = "data"; // Relative to the sysinspect root +pub static CFG_DEFAULT_ROOT: &str = "/etc/sysinspect"; +pub static CFG_DB: &str = "registry"; + +pub static CFG_MASTER_KEY_PUB: &str = "master.rsa.pub"; +pub static CFG_MASTER_KEY_PRI: &str = "master.rsa"; +pub static CFG_MINION_RSA_PUB: &str = "minion.rsa.pub"; +pub static CFG_MINION_RSA_PRV: &str = "minion.rsa"; #[derive(Debug, Serialize, Deserialize, Default, Clone)] pub struct MinionConfig { + /// Root directory where minion keeps all data. + /// Default: /etc/sysinspect — same as for master + root: Option, + + /// IP address of Master #[serde(rename = "master.ip")] master_ip: String, + /// Port of Master. Default: 4200 #[serde(rename = "master.port")] master_port: Option, + /// Port of Master's fileserver. Default: 4201 #[serde(rename = "master.fileserver.port")] master_fileserver_port: Option, } @@ -43,6 +61,11 @@ impl MinionConfig { pub fn fileserver(&self) -> String { format!("{}:{}", self.master_ip, self.master_fileserver_port.unwrap_or(DEFAULT_FILESERVER_PORT)) } + + /// Get minion root directory + pub fn root_dir(&self) -> PathBuf { + PathBuf::from(self.root.clone().unwrap_or(DEFAULT_SYSINSPECT_ROOT.to_string())) + } } #[derive(Debug, Serialize, Deserialize, Default, Clone)] @@ -97,4 +120,19 @@ impl MasterConfig { self.fileserver_port.unwrap_or(DEFAULT_FILESERVER_PORT) ) } + + /// Get default sysinspect root. For master it is always /etc/sysinspect + pub fn root_dir(&self) -> PathBuf { + PathBuf::from(DEFAULT_SYSINSPECT_ROOT.to_string()) + } + + /// Get fileserver root + pub fn fileserver_root(&self) -> PathBuf { + self.root_dir().join(CFG_FILESERVER_ROOT) + } + + /// Get minion keys store + pub fn keyman_root(&self) -> PathBuf { + self.root_dir().join(CFG_MINION_KEYS) + } } diff --git a/sysmaster/src/dataserv/fls.rs b/sysmaster/src/dataserv/fls.rs index 1d7b4d91..67c6174b 100644 --- a/sysmaster/src/dataserv/fls.rs +++ b/sysmaster/src/dataserv/fls.rs @@ -1,17 +1,17 @@ use actix_web::{rt::System, web, App, HttpResponse, HttpServer, Responder}; -use libsysinspect::{cfg::mmconf::MasterConfig, SysinspectError}; +use libsysinspect::{ + cfg::mmconf::{MasterConfig, CFG_FILESERVER_ROOT, DEFAULT_SYSINSPECT_ROOT}, + SysinspectError, +}; use std::{ fs, path::{Path, PathBuf}, thread, }; -use crate::registry::CFG_DEFAULT_ROOT; -static FILESERVER_ROOT_DIR: &str = "data"; - // Separate handler on every HTTP call async fn serve_file(path: web::Path, _cfg: web::Data) -> impl Responder { - let pth = Path::new(CFG_DEFAULT_ROOT).join(FILESERVER_ROOT_DIR).join(path.into_inner()); + let pth = Path::new(DEFAULT_SYSINSPECT_ROOT).join(CFG_FILESERVER_ROOT).join(path.into_inner()); log::debug!("Requested local file: {:?}", pth); if pth.is_file() { return HttpResponse::Ok().body(fs::read(pth).unwrap()); diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 06d8855e..cf84b454 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -33,7 +33,7 @@ pub struct SysMaster { impl SysMaster { pub fn new(cfg: MasterConfig) -> Result { let (tx, _) = broadcast::channel::>(100); - let mkr = MinionsKeyRegistry::new()?; + let mkr = MinionsKeyRegistry::new(cfg.keyman_root())?; Ok(SysMaster { cfg, broadcast: tx, mkr, to_drop: HashSet::default(), session: SessionKeeper::new(30) }) } diff --git a/sysmaster/src/registry/mkb.rs b/sysmaster/src/registry/mkb.rs index d5d477e4..bb2895bf 100644 --- a/sysmaster/src/registry/mkb.rs +++ b/sysmaster/src/registry/mkb.rs @@ -1,6 +1,8 @@ -use super::{CFG_DEFAULT_ROOT, CFG_MASTER_KEY_PRI, CFG_MASTER_KEY_PUB, CFG_MINION_KEYS}; use ::rsa::{RsaPrivateKey, RsaPublicKey}; -use libsysinspect::{rsa, SysinspectError}; +use libsysinspect::{ + cfg::mmconf::{CFG_MASTER_KEY_PRI, CFG_MASTER_KEY_PUB}, + rsa, SysinspectError, +}; use std::{collections::HashMap, fs, path::PathBuf}; /// Registered minion base. @@ -19,9 +21,8 @@ pub struct MinionsKeyRegistry { } impl MinionsKeyRegistry { - pub fn new() -> Result { - let mut reg = - MinionsKeyRegistry { root: PathBuf::from(CFG_DEFAULT_ROOT).join(CFG_MINION_KEYS), ..MinionsKeyRegistry::default() }; + pub fn new(root: PathBuf) -> Result { + let mut reg = MinionsKeyRegistry { root, ..MinionsKeyRegistry::default() }; reg.setup()?; Ok(reg) diff --git a/sysmaster/src/registry/mod.rs b/sysmaster/src/registry/mod.rs index 8c9b2b8a..6a4da956 100644 --- a/sysmaster/src/registry/mod.rs +++ b/sysmaster/src/registry/mod.rs @@ -11,12 +11,6 @@ use rec::MinionRecord; use serde_json::json; use sled::Db; -pub static CFG_DEFAULT_ROOT: &str = "/etc/sysinspect"; -pub static CFG_DB: &str = "registry"; -pub static CFG_MINION_KEYS: &str = "minion-keys"; -pub static CFG_MASTER_KEY_PUB: &str = "master.rsa.pub"; -pub static CFG_MASTER_KEY_PRI: &str = "master.rsa"; - pub struct MinionRegistry { conn: Db, } diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 0fed5c78..5bb85c03 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -36,14 +36,20 @@ impl SysMinion { let cfg = MinionConfig::new(cfp)?; let (rstm, wstm) = TcpStream::connect(cfg.master()).await.unwrap().into_split(); Ok(Arc::new(SysMinion { - cfg, + cfg: cfg.clone(), fingerprint, - kman: MinionRSAKeyManager::new(None)?, + kman: MinionRSAKeyManager::new(cfg.root_dir())?, rstm: Arc::new(Mutex::new(rstm)), wstm: Arc::new(Mutex::new(wstm)), })) } + /// Initialise minion. + /// This creates all directory structures if none etc. + fn init(&self) -> Result<(), SysinspectError> { + Ok(()) + } + pub fn as_ptr(self: &Arc) -> Arc { Arc::clone(self) } diff --git a/sysminion/src/rsa.rs b/sysminion/src/rsa.rs index f7f762b8..345204c5 100644 --- a/sysminion/src/rsa.rs +++ b/sysminion/src/rsa.rs @@ -4,13 +4,12 @@ RSA keys manager use std::{fs, path::PathBuf}; -use libsysinspect::SysinspectError; +use libsysinspect::{ + cfg::mmconf::{CFG_MINION_RSA_PRV, CFG_MINION_RSA_PUB}, + SysinspectError, +}; use rsa::{RsaPrivateKey, RsaPublicKey}; -static CFG_DEFAULT_MINION_ROOT: &str = "/etc/sysinspect"; -static CFG_MINION_RSA_PUB: &str = "minion.rsa.pub"; -static CFG_MINION_RSA_PRV: &str = "minion.rsa"; - #[derive(Debug, Default, Clone)] pub struct MinionRSAKeyManager { root: PathBuf, @@ -24,11 +23,8 @@ pub struct MinionRSAKeyManager { impl MinionRSAKeyManager { /// Initiate Minion's RSA key manager. Parameter `root` is /// optional, if configuration contains alternative Minion root. - pub fn new(root: Option) -> Result { - let mut keyman = MinionRSAKeyManager { - root: PathBuf::from(root.unwrap_or(CFG_DEFAULT_MINION_ROOT.to_string())), - ..Default::default() - }; + pub fn new(root: PathBuf) -> Result { + let mut keyman = MinionRSAKeyManager { root, ..Default::default() }; keyman.setup()?; Ok(keyman) From e1ed6e70e127e2793a19bb4c74d67094a5861d1f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 15:01:10 +0100 Subject: [PATCH 099/148] Update mmconf by adding models, functions and traits root dirs --- libsysinspect/src/cfg/mmconf.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/libsysinspect/src/cfg/mmconf.rs b/libsysinspect/src/cfg/mmconf.rs index 34591c70..e0ca860b 100644 --- a/libsysinspect/src/cfg/mmconf.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -3,17 +3,24 @@ use serde::{Deserialize, Serialize}; use serde_yaml::{from_str, from_value, Value}; use std::{fs, path::PathBuf}; +// Network pub static DEFAULT_ADDR: &str = "0.0.0.0"; pub static DEFAULT_PORT: u32 = 4200; pub static DEFAULT_FILESERVER_PORT: u32 = 4201; + +// Default directories pub static DEFAULT_SOCKET: &str = "/var/run/sysinspect-master.socket"; pub static DEFAULT_SYSINSPECT_ROOT: &str = "/etc/sysinspect"; +// All directories are relative to the sysinspect root pub static CFG_MINION_KEYS: &str = "minion-keys"; -pub static CFG_FILESERVER_ROOT: &str = "data"; // Relative to the sysinspect root -pub static CFG_DEFAULT_ROOT: &str = "/etc/sysinspect"; +pub static CFG_FILESERVER_ROOT: &str = "data"; +pub static CFG_MODELS_ROOT: &str = "models"; +pub static CFG_TRAITS_ROOT: &str = "traits"; +pub static CFG_TRAIT_FUNCTIONS_ROOT: &str = "functions"; pub static CFG_DB: &str = "registry"; +// Key names pub static CFG_MASTER_KEY_PUB: &str = "master.rsa.pub"; pub static CFG_MASTER_KEY_PRI: &str = "master.rsa"; pub static CFG_MINION_RSA_PUB: &str = "minion.rsa.pub"; @@ -66,6 +73,20 @@ impl MinionConfig { pub fn root_dir(&self) -> PathBuf { PathBuf::from(self.root.clone().unwrap_or(DEFAULT_SYSINSPECT_ROOT.to_string())) } + + /// Get root directory for models + pub fn models_dir(&self) -> PathBuf { + self.root_dir().join(CFG_MODELS_ROOT) + } + /// Get root directory for functions + pub fn functions_dir(&self) -> PathBuf { + self.root_dir().join(CFG_TRAIT_FUNCTIONS_ROOT) + } + + /// Get root directory for drop-in traits + pub fn traits_dir(&self) -> PathBuf { + self.root_dir().join(CFG_TRAITS_ROOT) + } } #[derive(Debug, Serialize, Deserialize, Default, Clone)] From 2b8361bff66790669afce316257b9d4fb240e6f7 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 15:02:01 +0100 Subject: [PATCH 100/148] Fix import order --- sysminion/src/rsa.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sysminion/src/rsa.rs b/sysminion/src/rsa.rs index 345204c5..7ba81447 100644 --- a/sysminion/src/rsa.rs +++ b/sysminion/src/rsa.rs @@ -2,13 +2,12 @@ RSA keys manager */ -use std::{fs, path::PathBuf}; - use libsysinspect::{ cfg::mmconf::{CFG_MINION_RSA_PRV, CFG_MINION_RSA_PUB}, SysinspectError, }; use rsa::{RsaPrivateKey, RsaPublicKey}; +use std::{fs, path::PathBuf}; #[derive(Debug, Default, Clone)] pub struct MinionRSAKeyManager { From 4d57b52a6db7338444bbc9664918447afaf5d3c4 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 15:02:07 +0100 Subject: [PATCH 101/148] Add minion init --- sysminion/src/minion.rs | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 5bb85c03..27247bd6 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -7,7 +7,7 @@ use libsysinspect::{ SysinspectError, }; use once_cell::sync::Lazy; -use std::{path::PathBuf, sync::Arc}; +use std::{fs, path::PathBuf, sync::Arc}; use tokio::net::{tcp::OwnedReadHalf, TcpStream}; use tokio::sync::Mutex; use tokio::{io::AsyncReadExt, sync::mpsc}; @@ -35,18 +35,49 @@ impl SysMinion { let cfg = MinionConfig::new(cfp)?; let (rstm, wstm) = TcpStream::connect(cfg.master()).await.unwrap().into_split(); - Ok(Arc::new(SysMinion { + let instance = SysMinion { cfg: cfg.clone(), fingerprint, kman: MinionRSAKeyManager::new(cfg.root_dir())?, rstm: Arc::new(Mutex::new(rstm)), wstm: Arc::new(Mutex::new(wstm)), - })) + }; + instance.init()?; + + Ok(Arc::new(instance)) } /// Initialise minion. /// This creates all directory structures if none etc. fn init(&self) -> Result<(), SysinspectError> { + log::info!("Initialising minion"); + // Place for models + if !self.cfg.models_dir().exists() { + log::debug!( + "Creating directory for the models at {}", + self.cfg.models_dir().as_os_str().to_str().unwrap_or_default() + ); + fs::create_dir_all(self.cfg.models_dir())?; + } + + // Place for traits.d + if !self.cfg.traits_dir().exists() { + log::debug!( + "Creating directory for the drop-in traits at {}", + self.cfg.traits_dir().as_os_str().to_str().unwrap_or_default() + ); + fs::create_dir_all(self.cfg.traits_dir())?; + } + + // Place for trait functions + if !self.cfg.functions_dir().exists() { + log::debug!( + "Creating directory for the custom trait functions at {}", + self.cfg.functions_dir().as_os_str().to_str().unwrap_or_default() + ); + fs::create_dir_all(self.cfg.functions_dir())?; + } + Ok(()) } From 32abdfc50e6fbb57e9744fbf2ae9f90f0967c224 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 16:09:00 +0100 Subject: [PATCH 102/148] Load custom traits from the minion --- libsysinspect/src/traits/mod.rs | 22 +------ libsysinspect/src/traits/systraits.rs | 88 +++++++++++++++++++++------ sysminion/src/minion.rs | 28 +++++++-- sysminion/src/proto.rs | 4 +- 4 files changed, 97 insertions(+), 45 deletions(-) diff --git a/libsysinspect/src/traits/mod.rs b/libsysinspect/src/traits/mod.rs index f120e5b9..ae2a8594 100644 --- a/libsysinspect/src/traits/mod.rs +++ b/libsysinspect/src/traits/mod.rs @@ -1,4 +1,5 @@ -mod systraits; +pub mod systraits; + use once_cell::sync::Lazy; use pest::Parser; use pest_derive::Parser; @@ -23,25 +24,6 @@ pub static HW_CPU_FREQ: &str = "hardware.cpu.frequency"; pub static HW_CPU_VENDOR: &str = "hardware.cpu.vendor"; pub static HW_CPU_CORES: &str = "hardware.cpu.cores"; -/* -Traits are system properties and attributes on which a minion is running. - -P.S. These are not Rust traits. :-) - */ - -/// System traits instance -static TRAITS: Lazy> = Lazy::new(|| Mutex::new(SystemTraits::new())); - -/// Returns a copy of initialised traits. -pub fn get_traits() -> SystemTraits { - let traits = &TRAITS; - if let Ok(traits) = traits.lock() { - return traits.to_owned(); - } - - SystemTraits::default() -} - #[derive(Parser)] #[grammar = "traits/traits_query.pest"] struct QueryParser; diff --git a/libsysinspect/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs index acea2943..aad86f63 100644 --- a/libsysinspect/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -1,24 +1,26 @@ +use crate::{ + cfg::mmconf::MinionConfig, + traits::{ + HW_CPU_BRAND, HW_CPU_CORES, HW_CPU_FREQ, HW_CPU_TOTAL, HW_CPU_VENDOR, HW_MEM, HW_SWAP, SYS_ID, SYS_NET_HOSTNAME, + SYS_OS_DISTRO, SYS_OS_KERNEL, SYS_OS_NAME, SYS_OS_VERSION, + }, + SysinspectError, +}; use indexmap::IndexMap; -use once_cell::sync::Lazy; use serde_json::{json, Value}; -use std::{fs, path::PathBuf}; -use tokio::sync::Mutex; - -use crate::traits::{ - HW_CPU_BRAND, HW_CPU_CORES, HW_CPU_FREQ, HW_CPU_TOTAL, HW_CPU_VENDOR, HW_MEM, HW_SWAP, SYS_ID, SYS_NET_HOSTNAME, - SYS_OS_DISTRO, SYS_OS_KERNEL, SYS_OS_NAME, SYS_OS_VERSION, -}; +use std::{collections::HashMap, fs, path::PathBuf}; /// SystemTraits contains a key/value of a system properties. #[derive(Debug, Clone, Default)] pub struct SystemTraits { data: IndexMap, + cfg: MinionConfig, } impl SystemTraits { - pub fn new() -> SystemTraits { + pub fn new(cfg: MinionConfig) -> SystemTraits { log::debug!("Initialising system traits"); - let mut traits = SystemTraits::default(); + let mut traits = SystemTraits { cfg, ..Default::default() }; traits.get_system(); traits.get_network(); traits.get_defined(); @@ -55,6 +57,17 @@ impl SystemTraits { self.data.keys().map(|s| s.to_string()).collect::>() } + /// Proxypass the error logging + fn proxy_log_error(result: Result, context: &str) -> Option { + match result { + Ok(v) => Some(v), + Err(err) => { + log::error!("{}: {}", context, err); + None + } + } + } + /// Read standard system traits fn get_system(&mut self) { log::debug!("Reading system traits data"); @@ -117,14 +130,51 @@ impl SystemTraits { } /// Read defined/configured static traits - fn get_defined(&self) { - log::debug!("Reading custon static traits data") + fn get_defined(&mut self) -> Result<(), SysinspectError> { + log::debug!("Reading custon static traits data"); + + for f in fs::read_dir(self.cfg.traits_dir())?.flatten() { + let fname = f.file_name(); + let fname = fname.to_str().unwrap_or_default(); + if fname.ends_with(".cfg") { + let content = Self::proxy_log_error( + fs::read_to_string(f.path()), + format!("Unable to read custom trait file at {}", fname).as_str(), + ) + .unwrap_or_default(); + + if content.is_empty() { + continue; + } + + let content: Option = + Self::proxy_log_error(serde_yaml::from_str(&content), "Custom trait file has broken YAML"); + + let content: Option = content.as_ref().and_then(|v| { + Self::proxy_log_error(serde_json::to_value(v), "Unable to convert existing YAML to JSON format") + }); + + if content.is_none() { + log::error!("Unable to load custom traits from {}", f.file_name().to_str().unwrap_or_default()); + continue; + } + + let content = content.as_ref().and_then(|v| { + Self::proxy_log_error( + serde_json::from_value::>(v.clone()), + "Unable to parse JSON", + ) + }); + + if let Some(content) = content { + for (k, v) in content { + self.put(k, json!(v)); + } + } else { + log::error!("Custom traits data is empty or in a wrong format"); + } + } + } + Ok(()) } } - -static _INSTANCE: Lazy> = Lazy::new(|| Mutex::new(SystemTraits::new())); - -/// Get traits -pub async fn get_traits() -> &'static Mutex { - &_INSTANCE -} diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 27247bd6..3e6c0b24 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -2,11 +2,12 @@ use crate::{proto, rsa::MinionRSAKeyManager}; use libsysinspect::{ cfg::{self, mmconf::MinionConfig}, proto::{errcodes::ProtoErrorCode, rqtypes::RequestType, MinionMessage, ProtoConversion}, - rsa, traits, + rsa, + traits::{self, systraits::SystemTraits}, util::dataconv, SysinspectError, }; -use once_cell::sync::Lazy; +use once_cell::sync::{Lazy, OnceCell}; use std::{fs, path::PathBuf, sync::Arc}; use tokio::net::{tcp::OwnedReadHalf, TcpStream}; use tokio::sync::Mutex; @@ -16,6 +17,19 @@ use uuid::Uuid; /// Session Id of the minion pub static MINION_SID: Lazy = Lazy::new(|| Uuid::new_v4().to_string()); +/* +Traits are system properties and attributes on which a minion is running. + +P.S. These are not Rust traits. :-) + */ + +/// System traits instance +static _TRAITS: OnceCell = OnceCell::new(); + +/// Returns a copy of initialised traits. +pub fn get_minion_traits() -> SystemTraits { + _TRAITS.get().expect("Traits are not initialised!?").to_owned() +} pub struct SysMinion { cfg: MinionConfig, @@ -34,6 +48,10 @@ impl SysMinion { } let cfg = MinionConfig::new(cfp)?; + + // Init traits + _TRAITS.get_or_init(|| SystemTraits::new(cfg.clone())); + let (rstm, wstm) = TcpStream::connect(cfg.master()).await.unwrap().into_split(); let instance = SysMinion { cfg: cfg.clone(), @@ -78,6 +96,8 @@ impl SysMinion { fs::create_dir_all(self.cfg.functions_dir())?; } + println!("Traits:\n{:#?}", get_minion_traits()); + Ok(()) } @@ -203,7 +223,7 @@ impl SysMinion { /// Send ehlo pub async fn send_ehlo(self: Arc) -> Result<(), SysinspectError> { let r = MinionMessage::new( - dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), + dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())), RequestType::Ehlo, MINION_SID.to_string(), ); @@ -216,7 +236,7 @@ impl SysMinion { /// Send registration request pub async fn send_registration(self: Arc, pbk_pem: String) -> Result<(), SysinspectError> { let r = - MinionMessage::new(dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); + MinionMessage::new(dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); log::info!("Registration request to {}", self.cfg.master()); self.request(r.sendable()?).await; diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index acccc1e5..bc96082b 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -1,5 +1,5 @@ pub mod msg { - use crate::minion::MINION_SID; + use crate::minion::{get_minion_traits, MINION_SID}; use libsysinspect::{ proto::{rqtypes::RequestType, MinionMessage}, traits, @@ -13,7 +13,7 @@ pub mod msg { /// Make pong message pub fn get_pong() -> Vec { let p = MinionMessage::new( - dataconv::as_str(traits::get_traits().get(traits::SYS_ID.to_string())), + dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())), RequestType::Pong, MINION_SID.to_string(), ); From c7303779ceb9774832907e551fbda445a23bb52d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 17:56:22 +0100 Subject: [PATCH 103/148] Add implementation of custom trait functions --- libsysinspect/src/traits/systraits.rs | 74 +++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/libsysinspect/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs index aad86f63..92949e8d 100644 --- a/libsysinspect/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -8,7 +8,13 @@ use crate::{ }; use indexmap::IndexMap; use serde_json::{json, Value}; -use std::{collections::HashMap, fs, path::PathBuf}; +use std::{ + collections::HashMap, + fs::{self, metadata}, + os::unix::fs::PermissionsExt, + path::PathBuf, + process::Command, +}; /// SystemTraits contains a key/value of a system properties. #[derive(Debug, Clone, Default)] @@ -23,7 +29,13 @@ impl SystemTraits { let mut traits = SystemTraits { cfg, ..Default::default() }; traits.get_system(); traits.get_network(); - traits.get_defined(); + if let Err(err) = traits.get_defined() { + log::error!("Unable to load custom traits: {err}"); + } + + if let Err(err) = traits.get_functions() { + log::error!("Unable to load trait functions: {err}"); + } traits } @@ -70,7 +82,7 @@ impl SystemTraits { /// Read standard system traits fn get_system(&mut self) { - log::debug!("Reading system traits data"); + log::info!("Loading system traits data"); let system = sysinfo::System::new_all(); // Common @@ -118,7 +130,7 @@ impl SystemTraits { /// Load network data fn get_network(&mut self) { - log::debug!("Reading network traits data"); + log::info!("Lading network traits data"); let net = sysinfo::Networks::new_with_refreshed_list(); for (ifs, data) in net.iter() { self.put(format!("system.net.{}.mac", ifs), json!(data.mac_address().to_string())); @@ -131,7 +143,7 @@ impl SystemTraits { /// Read defined/configured static traits fn get_defined(&mut self) -> Result<(), SysinspectError> { - log::debug!("Reading custon static traits data"); + log::debug!("Loading custom static traits data"); for f in fs::read_dir(self.cfg.traits_dir())?.flatten() { let fname = f.file_name(); @@ -177,4 +189,56 @@ impl SystemTraits { } Ok(()) } + + /// Load custom functions + fn get_functions(&mut self) -> Result<(), SysinspectError> { + log::info!("Loading trait functions"); + for f in fs::read_dir(self.cfg.functions_dir())?.flatten() { + let fname = f.path(); + let fname = fname.file_name().unwrap_or_default().to_str().unwrap_or_default(); + + log::info!("Calling function {fname}"); + + let is_exec = match fs::metadata(f.path()) { + Ok(m) => { + #[cfg(unix)] + { + m.permissions().mode() & 0o111 != 0 + } + } + Err(_) => false, + }; + + if is_exec { + let out = Command::new(f.path()).output()?; + if out.status.success() { + let data = Self::proxy_log_error( + String::from_utf8(out.stdout), + format!("Unable to load content from the function {}", fname).as_str(), + ); + if data.is_none() { + log::error!("Function {fname} returned no content"); + continue; + } + let data = data.unwrap_or_default(); + let data = Self::proxy_log_error( + serde_json::from_str::>(&data), + format!("Unable to parse JSON output from trait function at {fname}").as_str(), + ); + if let Some(data) = data { + for (k, v) in data { + self.put(k, json!(v)); + } + } else { + log::error!("Custom traits data is empty or in a wrong format"); + } + } else { + log::error!("Error running {fname}"); + } + } else { + log::warn!("Function {fname} is not an executable, skipping"); + } + } + Ok(()) + } } From 347ccf6e75dc07b9b5c5113cc85a9c879ffde4d1 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 17:57:09 +0100 Subject: [PATCH 104/148] Lintfix imports --- libsysinspect/src/traits/mod.rs | 6 +----- libsysinspect/src/traits/systraits.rs | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/libsysinspect/src/traits/mod.rs b/libsysinspect/src/traits/mod.rs index ae2a8594..31860461 100644 --- a/libsysinspect/src/traits/mod.rs +++ b/libsysinspect/src/traits/mod.rs @@ -1,12 +1,8 @@ pub mod systraits; -use once_cell::sync::Lazy; +use crate::SysinspectError; use pest::Parser; use pest_derive::Parser; -use std::sync::Mutex; -use systraits::SystemTraits; - -use crate::SysinspectError; /// Standard Traits pub static SYS_ID: &str = "system.id"; diff --git a/libsysinspect/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs index 92949e8d..49134ffa 100644 --- a/libsysinspect/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -10,7 +10,7 @@ use indexmap::IndexMap; use serde_json::{json, Value}; use std::{ collections::HashMap, - fs::{self, metadata}, + fs::{self}, os::unix::fs::PermissionsExt, path::PathBuf, process::Command, From 4261d6e8ae18041304b3e7db4eb97247780e56e8 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 18:22:38 +0100 Subject: [PATCH 105/148] Return sorted trait items --- libsysinspect/src/traits/systraits.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libsysinspect/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs index 49134ffa..0f14d5bf 100644 --- a/libsysinspect/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -66,7 +66,10 @@ impl SystemTraits { /// Return known trait items pub fn items(&self) -> Vec { - self.data.keys().map(|s| s.to_string()).collect::>() + let mut items = self.data.keys().map(|s| s.to_string()).collect::>(); + items.sort(); + + items } /// Proxypass the error logging From 9b5ba979c3ac8f08ede6b3999d666b99fe403f0f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 18:22:54 +0100 Subject: [PATCH 106/148] Move traits listing to debug realm --- sysminion/src/minion.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 3e6c0b24..f81305d7 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -8,7 +8,7 @@ use libsysinspect::{ SysinspectError, }; use once_cell::sync::{Lazy, OnceCell}; -use std::{fs, path::PathBuf, sync::Arc}; +use std::{fs, path::PathBuf, sync::Arc, vec}; use tokio::net::{tcp::OwnedReadHalf, TcpStream}; use tokio::sync::Mutex; use tokio::{io::AsyncReadExt, sync::mpsc}; @@ -96,7 +96,11 @@ impl SysMinion { fs::create_dir_all(self.cfg.functions_dir())?; } - println!("Traits:\n{:#?}", get_minion_traits()); + let mut out: Vec = vec![]; + for t in get_minion_traits().items() { + out.push(format!("{}: {}", t.to_owned(), dataconv::to_string(get_minion_traits().get(t)).unwrap_or_default())); + } + log::debug!("Minion traits:\n{}", out.join("\n")); Ok(()) } From 4e2ed978e8617171131e0824d856e6bd6010bf13 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 22:21:28 +0100 Subject: [PATCH 107/148] Update dependencies --- Cargo.lock | 1 + libsysinspect/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3e258ae6..41243ca3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1517,6 +1517,7 @@ checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", "hashbrown", + "serde", ] [[package]] diff --git a/libsysinspect/Cargo.toml b/libsysinspect/Cargo.toml index 135ae67f..6baed770 100644 --- a/libsysinspect/Cargo.toml +++ b/libsysinspect/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" base64 = "0.22.1" chrono = "0.4.38" colored = "2.1.0" -indexmap = "2.6.0" +indexmap = { version = "2.6.0", features = ["serde"] } lazy_static = "1.5.0" log = "0.4.22" nix = { version = "0.29.0", features = ["net", "user"] } From b4a35c856fea0e2293e180374aaf21d7aa859e6e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 12 Nov 2024 22:23:48 +0100 Subject: [PATCH 108/148] Lazymake a checksum of all traits --- libsysinspect/src/traits/systraits.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/libsysinspect/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs index 0f14d5bf..345c0dd5 100644 --- a/libsysinspect/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -8,6 +8,7 @@ use crate::{ }; use indexmap::IndexMap; use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; use std::{ collections::HashMap, fs::{self}, @@ -21,6 +22,7 @@ use std::{ pub struct SystemTraits { data: IndexMap, cfg: MinionConfig, + checksum: String, } impl SystemTraits { @@ -72,6 +74,21 @@ impl SystemTraits { items } + /// Return checksum of traits. + /// This is done by calculating checksum of the *keys*, as values can change on every restart, + /// e.g. IPv6 data, which is usually irrelevant or any other things that *meant* to change. + pub fn checksum(&mut self) -> String { + if !self.checksum.is_empty() { + return self.checksum.to_owned(); + } + + let mut keys = self.data.keys().map(|s| s.to_string()).collect::>(); + keys.sort(); + + self.checksum = format!("{:x}", Sha256::digest(keys.join("|").as_bytes())); + self.checksum.to_owned() + } + /// Proxypass the error logging fn proxy_log_error(result: Result, context: &str) -> Option { match result { From f9a70e1407261fae0793ead60d2d24478f39a87d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 13 Nov 2024 17:53:07 +0100 Subject: [PATCH 109/148] Add traits autosync --- libsysinspect/src/cfg/mmconf.rs | 6 ++ libsysinspect/src/proto/mod.rs | 8 ++- libsysinspect/src/traits/systraits.rs | 5 ++ sysmaster/src/master.rs | 51 +++++++++++++- sysmaster/src/registry/mod.rs | 63 +----------------- sysmaster/src/registry/mreg.rs | 95 +++++++++++++++++++++++++++ sysmaster/src/registry/rec.rs | 25 ++++++- sysminion/src/minion.rs | 18 ++++- 8 files changed, 203 insertions(+), 68 deletions(-) create mode 100644 sysmaster/src/registry/mreg.rs diff --git a/libsysinspect/src/cfg/mmconf.rs b/libsysinspect/src/cfg/mmconf.rs index e0ca860b..93ee7036 100644 --- a/libsysinspect/src/cfg/mmconf.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -14,6 +14,7 @@ pub static DEFAULT_SYSINSPECT_ROOT: &str = "/etc/sysinspect"; // All directories are relative to the sysinspect root pub static CFG_MINION_KEYS: &str = "minion-keys"; +pub static CFG_MINION_REGISTRY: &str = "minion-registry"; pub static CFG_FILESERVER_ROOT: &str = "data"; pub static CFG_MODELS_ROOT: &str = "models"; pub static CFG_TRAITS_ROOT: &str = "traits"; @@ -156,4 +157,9 @@ impl MasterConfig { pub fn keyman_root(&self) -> PathBuf { self.root_dir().join(CFG_MINION_KEYS) } + + /// Get minion registry + pub fn minion_registry_root(&self) -> PathBuf { + self.root_dir().join(CFG_MINION_REGISTRY) + } } diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 3cb344ec..9d67d3c9 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -68,6 +68,7 @@ impl MasterMessage { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MinionMessage { id: String, + sid: String, // Temporary session Id #[serde(rename = "r")] request: RequestType, @@ -82,7 +83,7 @@ pub struct MinionMessage { impl MinionMessage { /// Message constructor pub fn new(id: String, rtype: RequestType, data: String) -> MinionMessage { - MinionMessage { id, request: rtype, data, retcode: ProtoErrorCode::Undef as usize } + MinionMessage { id, request: rtype, data, retcode: ProtoErrorCode::Undef as usize, sid: "".to_string() } } /// Set return code @@ -103,6 +104,11 @@ impl MinionMessage { } } + /// Set Session Id + pub fn set_sid(&mut self, sid: String) { + self.sid = sid + } + /// Request type pub fn req_type(&self) -> &RequestType { &self.request diff --git a/libsysinspect/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs index 345c0dd5..2fda4589 100644 --- a/libsysinspect/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -261,4 +261,9 @@ impl SystemTraits { } Ok(()) } + + /// Convert the data to the JSON body + pub fn to_json_string(&self) -> Result { + Ok(serde_json::to_string(&json!(self.data))?) + } } diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index cf84b454..ec43f09a 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -2,6 +2,7 @@ use crate::{ dataserv::fls, registry::{ mkb::MinionsKeyRegistry, + mreg::MinionRegistry, session::{self, SessionKeeper}, }, }; @@ -10,7 +11,11 @@ use libsysinspect::{ proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; -use std::{collections::HashSet, path::Path, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, + sync::Arc, +}; use tokio::net::TcpListener; use tokio::select; use tokio::sync::{broadcast, mpsc}; @@ -26,6 +31,7 @@ pub struct SysMaster { cfg: MasterConfig, broadcast: broadcast::Sender>, mkr: MinionsKeyRegistry, + mreg: MinionRegistry, to_drop: HashSet, session: session::SessionKeeper, } @@ -34,7 +40,8 @@ impl SysMaster { pub fn new(cfg: MasterConfig) -> Result { let (tx, _) = broadcast::channel::>(100); let mkr = MinionsKeyRegistry::new(cfg.keyman_root())?; - Ok(SysMaster { cfg, broadcast: tx, mkr, to_drop: HashSet::default(), session: SessionKeeper::new(30) }) + let mreg = MinionRegistry::new(cfg.minion_registry_root())?; + Ok(SysMaster { cfg, broadcast: tx, mkr, to_drop: HashSet::default(), session: SessionKeeper::new(30), mreg }) } /// Open FIFO socket for command-line communication @@ -86,6 +93,17 @@ impl SysMaster { &mut self.mkr } + /// Request minion to sync its traits + fn msg_request_traits(&mut self, mid: String, sid: String) -> MasterMessage { + let mut m = MasterMessage::new(RequestType::Traits, sid); + let mut tgt = MinionTarget::new(); + tgt.add_minion_id(mid); + m.add_target(tgt); + m.set_retcode(ProtoErrorCode::Success); + + m + } + /// Already connected fn msg_already_connected(&mut self, mid: String, sid: String) -> MasterMessage { let mut m = MasterMessage::new(RequestType::Command, sid); @@ -178,6 +196,9 @@ impl SysMaster { } else { log::info!("{} connected successfully", c_id); guard.session.ping(&c_id, &c_payload); + _ = c_bcast + .send(guard.msg_request_traits(req.id().to_string(), c_payload).sendable().unwrap()); + log::info!("Syncing traits with minion at {}", c_id); } }); } @@ -197,6 +218,17 @@ impl SysMaster { ); }); } + + RequestType::Traits => { + log::debug!("Syncing traits from {}", req.id()); + let c_master = Arc::clone(&master); + let c_id = req.id().to_string(); + let c_payload = req.payload().to_string(); + tokio::spawn(async move { + let mut guard = c_master.lock().await; + guard.on_traits(c_id, c_payload).await; + }); + } _ => { log::error!("Minion sends unknown request type"); } @@ -209,6 +241,21 @@ impl SysMaster { }); } + pub async fn on_traits(&mut self, mid: String, payload: String) { + let traits = serde_json::from_str::>(&payload).unwrap_or_default(); + if !traits.is_empty() { + if let Err(err) = self.mreg.refresh(&mid, traits) { + log::error!("Unable to sync traits: {err}"); + } else { + log::info!("Traits added"); + } + } + //self.mreg.add(&mid, mrec); + //let x = serde_json::to_vec(&traits).unwrap_or_default(); + //let y = serde_json::from_slice::>(&x).unwrap(); + //println!("{:#?}", y); + } + pub async fn do_fifo(master: Arc>) { log::trace!("Init local command channel"); tokio::spawn(async move { diff --git a/sysmaster/src/registry/mod.rs b/sysmaster/src/registry/mod.rs index 6a4da956..34ecf763 100644 --- a/sysmaster/src/registry/mod.rs +++ b/sysmaster/src/registry/mod.rs @@ -3,67 +3,6 @@ Minion registry. It contains minion tasks, traits, location and other data */ pub mod mkb; +pub mod mreg; pub mod rec; pub mod session; - -use libsysinspect::SysinspectError; -use rec::MinionRecord; -use serde_json::json; -use sled::Db; - -pub struct MinionRegistry { - conn: Db, -} - -impl MinionRegistry { - pub fn new(pth: &str) -> Result { - Ok(MinionRegistry { - conn: match sled::open(pth) { - Ok(db) => db, - Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), - }, - }) - } - - pub fn add(&mut self, mid: &str, mrec: MinionRecord) -> Result<(), SysinspectError> { - if let Err(err) = self.conn.insert(mid, json!(mrec).to_string().as_bytes().to_vec()) { - return Err(SysinspectError::MasterGeneralError(format!("{err}"))); - } - - Ok(()) - } - - pub fn get(&mut self, mid: &str) -> Result, SysinspectError> { - let data = match self.conn.get(mid) { - Ok(data) => data, - Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), - }; - - if let Some(data) = data { - return Ok(Some(match String::from_utf8(data.to_vec()) { - Ok(data) => match serde_json::from_str::(&data) { - Ok(mrec) => mrec, - Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), - }, - Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), - })); - } - - Ok(None) - } - - pub fn remove(&mut self, mid: &str) -> Result<(), SysinspectError> { - let contains = match self.conn.contains_key(mid) { - Ok(res) => res, - Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), - }; - - if contains { - if let Err(err) = self.conn.remove(mid) { - return Err(SysinspectError::MasterGeneralError(format!("{err}"))); - }; - } - - Ok(()) - } -} diff --git a/sysmaster/src/registry/mreg.rs b/sysmaster/src/registry/mreg.rs new file mode 100644 index 00000000..08333ba4 --- /dev/null +++ b/sysmaster/src/registry/mreg.rs @@ -0,0 +1,95 @@ +use super::rec::MinionRecord; +use libsysinspect::SysinspectError; +use serde_json::{json, Value}; +use sled::Db; +use std::{collections::HashMap, fs, path::PathBuf}; + +#[derive(Debug)] +pub struct MinionRegistry { + conn: Db, +} + +impl MinionRegistry { + pub fn new(pth: PathBuf) -> Result { + if !pth.exists() { + fs::create_dir_all(&pth)?; + } + + Ok(MinionRegistry { + conn: match sled::open(pth) { + Ok(db) => db, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }, + }) + } + + /// Add or update traits + pub fn refresh(&mut self, mid: &str, traits: HashMap) -> Result<(), SysinspectError> { + match self.conn.contains_key(mid) { + Ok(exists) => { + if exists { + if let Err(err) = self.conn.remove(mid) { + return Err(SysinspectError::MasterGeneralError(format!( + "Unable to remove previous data for {mid} from the database: {err}" + ))); + } else { + log::debug!("Traits for {mid} pre-removed"); + } + } + } + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("Unable to access the database: {err}"))), + }; + + self.add(mid, MinionRecord::new(mid.to_string(), traits))?; + + Ok(()) + } + + fn add(&mut self, mid: &str, mrec: MinionRecord) -> Result<(), SysinspectError> { + if let Err(err) = self.conn.insert(mid, json!(mrec).to_string().as_bytes().to_vec()) { + return Err(SysinspectError::MasterGeneralError(format!("{err}"))); + } + + Ok(()) + } + + pub fn get(&mut self, mid: &str) -> Result, SysinspectError> { + let data = match self.conn.get(mid) { + Ok(data) => data, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }; + + if let Some(data) = data { + return Ok(Some(match String::from_utf8(data.to_vec()) { + Ok(data) => match serde_json::from_str::(&data) { + Ok(mrec) => mrec, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + })); + } + + Ok(None) + } + + pub fn remove(&mut self, mid: &str) -> Result<(), SysinspectError> { + let contains = match self.conn.contains_key(mid) { + Ok(res) => res, + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), + }; + + if contains { + if let Err(err) = self.conn.remove(mid) { + return Err(SysinspectError::MasterGeneralError(format!("{err}"))); + }; + } + + Ok(()) + } + + /// Select minions by trait criterias + pub fn select(&self, traits: HashMap) -> Vec { + // XXX: Pretty much crude-dumb implementation which doesn't scale. But good enough for 0.x version. :-) + vec![] + } +} diff --git a/sysmaster/src/registry/rec.rs b/sysmaster/src/registry/rec.rs index a5ba77f9..d97d3294 100644 --- a/sysmaster/src/registry/rec.rs +++ b/sysmaster/src/registry/rec.rs @@ -1,4 +1,25 @@ +use std::{collections::HashMap, default}; + use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct MinionRecord { + id: String, + traits: HashMap, +} + +impl MinionRecord { + pub fn new(id: String, traits: HashMap) -> Self { + MinionRecord { id, traits } + } + + /// Check if the record matches the value + pub fn matches(&self, attr: &str, v: Value) -> bool { + self.traits.get(attr).and_then(|f| Some(f.eq(&v))).unwrap_or(false) + } -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] -pub struct MinionRecord {} + pub fn get_traits(&self) -> &HashMap { + &self.traits + } +} diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index f81305d7..4cbab003 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -109,6 +109,11 @@ impl SysMinion { Arc::clone(self) } + /// Get current minion Id + fn get_minion_id(&self) -> String { + dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())) + } + /// Talk-back to the master async fn request(&self, msg: Vec) { let mut stm = self.wstm.lock().await; @@ -200,6 +205,9 @@ impl SysMinion { } RequestType::Traits => { log::debug!("Master requests traits"); + if let Err(err) = cls.as_ptr().send_traits().await { + log::error!("Unable to send traits: {err}"); + } } RequestType::AgentUnknown => { let pbk_pem = msg.payload(); // Expected PEM RSA pub key @@ -224,13 +232,21 @@ impl SysMinion { Ok(()) } + pub async fn send_traits(self: Arc) -> Result<(), SysinspectError> { + let mut r = MinionMessage::new(self.get_minion_id(), RequestType::Traits, get_minion_traits().to_json_string()?); + r.set_sid(MINION_SID.to_string()); + self.request(r.sendable().unwrap()).await; // XXX: make a better error handling for Tokio + Ok(()) + } + /// Send ehlo pub async fn send_ehlo(self: Arc) -> Result<(), SysinspectError> { - let r = MinionMessage::new( + let mut r = MinionMessage::new( dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())), RequestType::Ehlo, MINION_SID.to_string(), ); + r.set_sid(MINION_SID.to_string()); log::info!("Ehlo on {}", self.cfg.master()); self.request(r.sendable()?).await; From dc06ee079421c6143d04a1c8a6284d42b3ff938c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 13 Nov 2024 17:54:58 +0100 Subject: [PATCH 110/148] Simplify expression --- sysmaster/src/registry/rec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sysmaster/src/registry/rec.rs b/sysmaster/src/registry/rec.rs index d97d3294..d3527aaa 100644 --- a/sysmaster/src/registry/rec.rs +++ b/sysmaster/src/registry/rec.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, default}; +use std::collections::HashMap; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -16,7 +16,7 @@ impl MinionRecord { /// Check if the record matches the value pub fn matches(&self, attr: &str, v: Value) -> bool { - self.traits.get(attr).and_then(|f| Some(f.eq(&v))).unwrap_or(false) + self.traits.get(attr).map(|f| f.eq(&v)).unwrap_or(false) } pub fn get_traits(&self) -> &HashMap { From 6b5529a1f952748b242b9135849f431457222d40 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 14 Nov 2024 15:01:55 +0100 Subject: [PATCH 111/148] Use a separate BTree with its own namespace for the minions, implement query by traits (crude/simple) --- sysmaster/src/registry/mreg.rs | 70 +++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/sysmaster/src/registry/mreg.rs b/sysmaster/src/registry/mreg.rs index 08333ba4..f67bf78b 100644 --- a/sysmaster/src/registry/mreg.rs +++ b/sysmaster/src/registry/mreg.rs @@ -1,9 +1,11 @@ use super::rec::MinionRecord; use libsysinspect::SysinspectError; use serde_json::{json, Value}; -use sled::Db; +use sled::{Db, Tree}; use std::{collections::HashMap, fs, path::PathBuf}; +const DB_MINIONS: &str = "minions"; + #[derive(Debug)] pub struct MinionRegistry { conn: Db, @@ -23,12 +25,21 @@ impl MinionRegistry { }) } + fn get_tree(&self, tid: &str) -> Result { + let tree = self.conn.open_tree(tid); + if let Err(err) = tree { + return Err(SysinspectError::MasterGeneralError(format!("Unable to open {tid} database: {err}"))); + } + Ok(tree.unwrap()) + } + /// Add or update traits pub fn refresh(&mut self, mid: &str, traits: HashMap) -> Result<(), SysinspectError> { - match self.conn.contains_key(mid) { + let minions = self.get_tree(DB_MINIONS)?; + match minions.contains_key(mid) { Ok(exists) => { if exists { - if let Err(err) = self.conn.remove(mid) { + if let Err(err) = minions.remove(mid) { return Err(SysinspectError::MasterGeneralError(format!( "Unable to remove previous data for {mid} from the database: {err}" ))); @@ -46,7 +57,8 @@ impl MinionRegistry { } fn add(&mut self, mid: &str, mrec: MinionRecord) -> Result<(), SysinspectError> { - if let Err(err) = self.conn.insert(mid, json!(mrec).to_string().as_bytes().to_vec()) { + let minions = self.get_tree(DB_MINIONS)?; + if let Err(err) = minions.insert(mid, json!(mrec).to_string().as_bytes().to_vec()) { return Err(SysinspectError::MasterGeneralError(format!("{err}"))); } @@ -54,7 +66,8 @@ impl MinionRegistry { } pub fn get(&mut self, mid: &str) -> Result, SysinspectError> { - let data = match self.conn.get(mid) { + let minions = self.get_tree(DB_MINIONS)?; + let data = match minions.get(mid) { Ok(data) => data, Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), }; @@ -73,13 +86,14 @@ impl MinionRegistry { } pub fn remove(&mut self, mid: &str) -> Result<(), SysinspectError> { - let contains = match self.conn.contains_key(mid) { + let minions = self.get_tree(DB_MINIONS)?; + let contains = match minions.contains_key(mid) { Ok(res) => res, Err(err) => return Err(SysinspectError::MasterGeneralError(format!("{err}"))), }; if contains { - if let Err(err) = self.conn.remove(mid) { + if let Err(err) = minions.remove(mid) { return Err(SysinspectError::MasterGeneralError(format!("{err}"))); }; } @@ -88,8 +102,46 @@ impl MinionRegistry { } /// Select minions by trait criterias - pub fn select(&self, traits: HashMap) -> Vec { + pub fn select(&self, traits: HashMap) -> Result, SysinspectError> { + let minions = self.get_tree(DB_MINIONS)?; + let mut mns: Vec = Vec::default(); + // XXX: Pretty much crude-dumb implementation which doesn't scale. But good enough for 0.x version. :-) - vec![] + for entry in minions.iter() { + match entry { + Ok((k_ent, v_ent)) => { + let mid = String::from_utf8(k_ent.to_vec()).unwrap_or_default(); + let mrec = serde_json::from_str::(&String::from_utf8(v_ent.to_vec()).unwrap_or_default()); + let mrec = match mrec { + Ok(mrec) => mrec, + Err(err) => { + return Err(SysinspectError::MasterGeneralError(format!("Unable to read minion record: {err}"))) + } + }; + + let mut matches = false; + for (kreq, vreq) in &traits { + if let Some(v) = mrec.get_traits().get(kreq) { + if vreq.eq(v) { + matches = true; + } else { + matches = true; + break; + } + } else { + matches = false; + break; + } + } + + if matches { + mns.push(mid); + } + } + Err(err) => return Err(SysinspectError::MasterGeneralError(format!("Minion database seems corrupt: {err}"))), + }; + } + + Ok(mns) } } From 6b56e2e883c1385580e2f6707d4b3f6a4b61bb39 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:32:25 +0100 Subject: [PATCH 112/148] Add quoted values and dotted keys --- libsysinspect/src/traits/mod.rs | 71 ++++++++++++++++------ libsysinspect/src/traits/traits_query.pest | 11 ++-- 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/libsysinspect/src/traits/mod.rs b/libsysinspect/src/traits/mod.rs index 31860461..66a19a1a 100644 --- a/libsysinspect/src/traits/mod.rs +++ b/libsysinspect/src/traits/mod.rs @@ -1,8 +1,12 @@ pub mod systraits; +use std::collections::HashMap; + use crate::SysinspectError; use pest::Parser; use pest_derive::Parser; +use serde_json::Value; +use systraits::SystemTraits; /// Standard Traits pub static SYS_ID: &str = "system.id"; @@ -34,29 +38,60 @@ struct QueryParser; /// [[foo, bar], [baz]] /// /// Each inner array should be treated with AND operator. -pub fn get_traits_query(input: &str) -> Result>, SysinspectError> { +pub fn parse_traits_query(input: &str) -> Result>, SysinspectError> { + let pairs = QueryParser::parse(Rule::expression, input) + .map_err(|err| SysinspectError::ModelDSLError(format!("Invalid query: {err}")))?; + let mut out = Vec::new(); - let mut pairs = match QueryParser::parse(Rule::expression, input) { - Ok(prs) => prs, - Err(err) => return Err(SysinspectError::ModelDSLError(format!("Invalid query: {err}"))), - }; - - let expr = match pairs.next() { - Some(expr) => expr, - None => return Ok(out), - }; - - for grp in expr.into_inner() { - if grp.as_rule() == Rule::group { - let mut terms = Vec::new(); - for t_pair in grp.into_inner() { - if t_pair.as_rule() == Rule::term { - terms.push(t_pair.as_str().to_string()); + + for expr in pairs { + if expr.as_rule() == Rule::expression { + for group_pair in expr.into_inner() { + if group_pair.as_rule() == Rule::group { + let mut terms = Vec::new(); + for term_pair in group_pair.into_inner() { + if term_pair.as_rule() == Rule::term { + terms.push(term_pair.as_str().trim().to_string()); + } + } + out.push(terms); } } - out.push(terms); } } Ok(out) } + +/// Parse trait query to trait typed (JSON) query +pub fn to_typed_query(qt: Vec>) -> Result>>, SysinspectError> { + let mut out: Vec>> = Vec::default(); + for and_op in qt { + let mut out_op: Vec> = Vec::default(); + for op in and_op { + let x = op.replace(":", ": "); + match serde_yaml::from_str::>(&x) { + Ok(v) => out_op.push(v), + Err(e) => return Err(SysinspectError::MinionGeneralError(format!("Broken traits query: {e}"))), + }; + } + out.push(out_op); + } + Ok(out) +} + +pub fn matches_traits(qt: Vec>>, traits: SystemTraits) -> bool { + let mut or_op_c: Vec = Vec::default(); + for and_op in qt { + let mut and_op_c: Vec = Vec::default(); + for ophm in and_op { + // op hashmap has always just one key and one value + for (opk, opv) in ophm { + and_op_c.push(traits.get(&opk).and_then(|x| Some(x.eq(&opv))).unwrap_or(false)); + } + } + or_op_c.push(!and_op_c.contains(&false)); + } + + or_op_c.contains(&true) +} diff --git a/libsysinspect/src/traits/traits_query.pest b/libsysinspect/src/traits/traits_query.pest index ca11d075..9c543a3c 100644 --- a/libsysinspect/src/traits/traits_query.pest +++ b/libsysinspect/src/traits/traits_query.pest @@ -1,12 +1,11 @@ -// src/query.pest - -// Define silent whitespace rule to skip over spaces and tabs WHITESPACE = _{ " " | "\t" } -// Define a term as one or more alphanumeric characters -term = @{ ASCII_ALPHANUMERIC+ } +// Define a term as one or more alphanumeric characters, or a quoted value +term = { key ~ ":" ~ value } +key = @{ (ASCII_ALPHANUMERIC | "-" | "_" | ".")+ } +value = @{ (ASCII_ALPHANUMERIC | "-" | "_")+ | quoted_value } +quoted_value = @{ "\"" ~ (!"\"" ~ ANY)* ~ "\"" } -// Define "and" and "or" operators as silent rules and_op = _{ "and" } or_op = _{ "or" } From b5c2decc1156714bd5208f4e750643d7969056f1 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:33:32 +0100 Subject: [PATCH 113/148] Send in main CLI scheme and query --- src/clidef.rs | 10 ++++++++-- src/main.rs | 13 ++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/clidef.rs b/src/clidef.rs index f1f2ae63..56bf0c80 100644 --- a/src/clidef.rs +++ b/src/clidef.rs @@ -20,11 +20,17 @@ pub fn cli(version: &'static str) -> Command { // Sysinspect .next_help_heading("Main") .arg( - Arg::new("query") - .help("Network query") + Arg::new("scheme") + .help("Specify scheme that needs to be requested (model:// or state://)") .required(false) .index(1) ) + .arg( + Arg::new("query") + .help("Minions to query") + .required(false) + .index(2) + ) .arg( Arg::new("traits") .short('t') diff --git a/src/main.rs b/src/main.rs index 6771f48e..a72f4af0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,10 +26,11 @@ fn print_event_handlers() { } /// Call master via FIFO -fn call_master_fifo(msg: &str, fifo: &str) -> Result<(), SysinspectError> { - OpenOptions::new().write(true).open(fifo)?.write_all(format!("{}\n", msg).as_bytes())?; +fn call_master_fifo(model: &str, query: &str, traits: Option<&String>, fifo: &str) -> Result<(), SysinspectError> { + let payload = format!("{model};{query};{}\n", traits.unwrap_or(&"".to_string())); + OpenOptions::new().write(true).open(fifo)?.write_all(payload.as_bytes())?; - log::debug!("Message sent to FIFO: {}", msg); + log::debug!("Message sent to the master via FIFO: {:?}", payload); Ok(()) } @@ -95,8 +96,10 @@ fn main() { } }; - if let Some(query) = params.get_one::("query") { - if let Err(err) = call_master_fifo(query, &cfg.socket()) { + if let Some(model) = params.get_one::("scheme") { + let query = params.get_one::("query"); + let traits = params.get_one::("traits"); + if let Err(err) = call_master_fifo(model, query.unwrap_or(&"".to_string()), traits, &cfg.socket()) { log::error!("Cannot reach master: {err}"); } } else if let Some(mpath) = params.get_one::("model") { From a7923ad8cff5f54fb18c097e9c7c65042b937091 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:33:55 +0100 Subject: [PATCH 114/148] Update deps --- Cargo.lock | 129 ++++++++++++++++++++++--------------------- sysmaster/Cargo.toml | 1 + sysminion/Cargo.toml | 2 + 3 files changed, 69 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41243ca3..d6abb42e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,7 +65,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -182,7 +182,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -202,9 +202,9 @@ checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" @@ -406,7 +406,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.85", + "syn 2.0.87", "which", ] @@ -499,9 +499,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "jobserver", "libc", @@ -566,18 +566,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstream", "anstyle", @@ -587,9 +587,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "cmake" @@ -657,9 +657,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -710,9 +710,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", @@ -753,7 +753,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -786,7 +786,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -830,7 +830,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1034,7 +1034,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1149,9 +1149,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "hermit-abi" @@ -1479,7 +1479,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1638,9 +1638,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "libloading" @@ -1996,7 +1996,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2137,7 +2137,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2235,7 +2235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2284,7 +2284,7 @@ dependencies = [ "flate2", "hex", "procfs-core", - "rustix 0.38.39", + "rustix 0.38.40", ] [[package]] @@ -2400,9 +2400,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -2549,9 +2549,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.39" +version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" +checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ "bitflags 2.6.0", "errno", @@ -2689,22 +2689,22 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2895,9 +2895,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -2921,7 +2921,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2977,6 +2977,7 @@ dependencies = [ "libsysinspect", "log", "rand", + "regex", "rsa", "rustls", "rustls-pemfile", @@ -2996,11 +2997,13 @@ dependencies = [ "clap", "colored", "ed25519-dalek", + "glob", "indexmap", "libsysinspect", "log", "once_cell", "rand", + "regex", "reqwest", "rsa", "rustls", @@ -3043,7 +3046,7 @@ dependencies = [ "cfg-if", "fastrand", "once_cell", - "rustix 0.38.39", + "rustix 0.38.40", "windows-sys 0.59.0", ] @@ -3083,22 +3086,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3168,7 +3171,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3382,7 +3385,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -3416,7 +3419,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3446,7 +3449,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.39", + "rustix 0.38.40", ] [[package]] @@ -3519,7 +3522,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3530,7 +3533,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3752,7 +3755,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "synstructure", ] @@ -3774,7 +3777,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3794,7 +3797,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "synstructure", ] @@ -3815,7 +3818,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3837,7 +3840,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] diff --git a/sysmaster/Cargo.toml b/sysmaster/Cargo.toml index 701b4323..82a4957c 100644 --- a/sysmaster/Cargo.toml +++ b/sysmaster/Cargo.toml @@ -33,3 +33,4 @@ sled = "0.34.7" rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } uuid = { version = "1.11.0", features = ["v4"] } actix-web = "4.9.0" +regex = "1.11.1" diff --git a/sysminion/Cargo.toml b/sysminion/Cargo.toml index c8dee472..a5f43c0d 100644 --- a/sysminion/Cargo.toml +++ b/sysminion/Cargo.toml @@ -32,3 +32,5 @@ sysinfo = { version = "0.32.0", features = ["linux-tmpfs"] } rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } uuid = { version = "1.11.0", features = ["v4"] } reqwest = "0.12.9" +regex = "1.11.1" +glob = "0.3.1" From f2584b791913220f557d9dcc24035d7f7d9cd356 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:35:57 +0100 Subject: [PATCH 115/148] Replace string value with the reference --- libsysinspect/src/traits/systraits.rs | 8 ++++---- sysminion/src/minion.rs | 11 ++++++----- sysminion/src/proto.rs | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/libsysinspect/src/traits/systraits.rs b/libsysinspect/src/traits/systraits.rs index 2fda4589..99a23db2 100644 --- a/libsysinspect/src/traits/systraits.rs +++ b/libsysinspect/src/traits/systraits.rs @@ -48,17 +48,17 @@ impl SystemTraits { } /// Get a trait value in JSON - pub fn get(&self, path: String) -> Option { - self.data.get(&path).cloned() + pub fn get(&self, path: &str) -> Option { + self.data.get(path).cloned() } /// Check if trait is present - pub fn has(&self, path: String) -> bool { + pub fn has(&self, path: &str) -> bool { self.get(path).is_some() } /// Check if trait matches the requested value. - pub fn matches(&self, path: String, v: Value) -> bool { + pub fn matches(&self, path: &str, v: Value) -> bool { if let Some(t) = self.get(path) { return t.eq(&v); } diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 4cbab003..1e2b244b 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,13 +1,14 @@ use crate::{proto, rsa::MinionRSAKeyManager}; use libsysinspect::{ cfg::{self, mmconf::MinionConfig}, - proto::{errcodes::ProtoErrorCode, rqtypes::RequestType, MinionMessage, ProtoConversion}, + proto::{errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, ProtoConversion}, rsa, traits::{self, systraits::SystemTraits}, util::dataconv, SysinspectError, }; use once_cell::sync::{Lazy, OnceCell}; +use regex::Regex; use std::{fs, path::PathBuf, sync::Arc, vec}; use tokio::net::{tcp::OwnedReadHalf, TcpStream}; use tokio::sync::Mutex; @@ -98,7 +99,7 @@ impl SysMinion { let mut out: Vec = vec![]; for t in get_minion_traits().items() { - out.push(format!("{}: {}", t.to_owned(), dataconv::to_string(get_minion_traits().get(t)).unwrap_or_default())); + out.push(format!("{}: {}", t.to_owned(), dataconv::to_string(get_minion_traits().get(&t)).unwrap_or_default())); } log::debug!("Minion traits:\n{}", out.join("\n")); @@ -111,7 +112,7 @@ impl SysMinion { /// Get current minion Id fn get_minion_id(&self) -> String { - dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())) + dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())) } /// Talk-back to the master @@ -242,7 +243,7 @@ impl SysMinion { /// Send ehlo pub async fn send_ehlo(self: Arc) -> Result<(), SysinspectError> { let mut r = MinionMessage::new( - dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())), + dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())), RequestType::Ehlo, MINION_SID.to_string(), ); @@ -256,7 +257,7 @@ impl SysMinion { /// Send registration request pub async fn send_registration(self: Arc, pbk_pem: String) -> Result<(), SysinspectError> { let r = - MinionMessage::new(dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); + MinionMessage::new(dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); log::info!("Registration request to {}", self.cfg.master()); self.request(r.sendable()?).await; diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index bc96082b..4442ab2c 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -13,7 +13,7 @@ pub mod msg { /// Make pong message pub fn get_pong() -> Vec { let p = MinionMessage::new( - dataconv::as_str(get_minion_traits().get(traits::SYS_ID.to_string())), + dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())), RequestType::Pong, MINION_SID.to_string(), ); From 58bd5bf70354a370905d2e14b3f8ef1e2993d1c9 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:36:43 +0100 Subject: [PATCH 116/148] XXX: Probably won't be needed --- sysminion/src/minion.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 1e2b244b..5ac6e3fa 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -136,6 +136,7 @@ impl SysMinion { } } + /* pub async fn do_qmsg(self: Arc) { let (_w_chan, mut r_chan) = mpsc::channel(100); let cls = Arc::new(self.clone()); @@ -145,6 +146,7 @@ impl SysMinion { } }); } + */ pub async fn do_proto(self: Arc) -> Result<(), SysinspectError> { let rstm = Arc::clone(&self.rstm); From c73dee22a0f862309437437d501db0ddaa1c6ed6 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:37:28 +0100 Subject: [PATCH 117/148] Add command dispatcher --- sysmaster/src/master.rs | 60 +++++++++++++++++++------------ sysminion/src/minion.rs | 80 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 23 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index ec43f09a..00c6a41a 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -11,6 +11,7 @@ use libsysinspect::{ proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; +use regex::Regex; use std::{ collections::{HashMap, HashSet}, path::Path, @@ -93,12 +94,32 @@ impl SysMaster { &mut self.mkr } + fn msg_query(&mut self, payload: &str) -> Option { + let query = payload.split(";").map(|s| s.to_string()).collect::>(); + + if let [scheme, query, traits] = query.as_slice() { + let mut tgt = MinionTarget::default(); + tgt.set_scheme(scheme); + tgt.set_traits_query(traits); + for hostname in query.split(",") { + tgt.add_hostname(hostname); + } + + let mut m = MasterMessage::new(RequestType::Command, "SID or something".to_string()); + m.set_target(tgt); + m.set_retcode(ProtoErrorCode::Success); + + return Some(m); + } + + None + } + /// Request minion to sync its traits fn msg_request_traits(&mut self, mid: String, sid: String) -> MasterMessage { - let mut m = MasterMessage::new(RequestType::Traits, sid); - let mut tgt = MinionTarget::new(); - tgt.add_minion_id(mid); - m.add_target(tgt); + let mut m = MasterMessage::new(RequestType::Traits, sid.clone()); + let mut tgt = MinionTarget::new(&mid, &sid); + m.set_target(tgt); m.set_retcode(ProtoErrorCode::Success); m @@ -106,10 +127,8 @@ impl SysMaster { /// Already connected fn msg_already_connected(&mut self, mid: String, sid: String) -> MasterMessage { - let mut m = MasterMessage::new(RequestType::Command, sid); - let mut tgt = MinionTarget::new(); - tgt.add_minion_id(mid); - m.add_target(tgt); + let mut m = MasterMessage::new(RequestType::Command, sid.clone()); + m.set_target(MinionTarget::new(&mid, &sid)); m.set_retcode(ProtoErrorCode::AlreadyConnected); m @@ -118,9 +137,7 @@ impl SysMaster { /// Bounce message fn msg_not_registered(&mut self, mid: String) -> MasterMessage { let mut m = MasterMessage::new(RequestType::AgentUnknown, self.mkr().get_master_key_pem().clone().unwrap().to_string()); - let mut tgt = MinionTarget::new(); - tgt.add_minion_id(mid); - m.add_target(tgt); + m.set_target(MinionTarget::new(&mid, "")); m.set_retcode(ProtoErrorCode::Success); m @@ -129,9 +146,7 @@ impl SysMaster { /// Accept registration fn msg_registered(&self, mid: String, msg: &str) -> MasterMessage { let mut m = MasterMessage::new(RequestType::Reconnect, msg.to_string()); // XXX: Should it be already encrypted? - let mut tgt = MinionTarget::new(); - tgt.add_minion_id(mid); - m.add_target(tgt); + m.set_target(MinionTarget::new(&mid, "")); m.set_retcode(ProtoErrorCode::Success); m @@ -250,10 +265,6 @@ impl SysMaster { log::info!("Traits added"); } } - //self.mreg.add(&mid, mrec); - //let x = serde_json::to_vec(&traits).unwrap_or_default(); - //let y = serde_json::from_slice::>(&x).unwrap(); - //println!("{:#?}", y); } pub async fn do_fifo(master: Arc>) { @@ -271,9 +282,12 @@ impl SysMaster { select! { line = lines.next_line() => { match line { - Ok(Some(message)) => { - log::info!("Broadcasting FIFO message to clients: {}", message); - let _ = bcast.send(message.into_bytes()); + Ok(Some(payload)) => { + log::info!("Querying minions: {}", payload); + if let Some(msg) = master.lock().await.msg_query(&payload) { + log::debug!("{:#?}", msg); + let _ = bcast.send(msg.sendable().unwrap()); + } } Ok(None) => break, // End of file, re-open the FIFO Err(e) => { @@ -301,9 +315,9 @@ impl SysMaster { loop { _ = time::sleep(Duration::from_secs(5)).await; let mut p = MasterMessage::new(RequestType::Ping, "".to_string()); - let mut t = MinionTarget::new(); + let mut t = MinionTarget::default(); t.add_hostname("*"); - p.add_target(t); + p.set_target(t); let _ = bcast.send(p.sendable().unwrap()); } }); diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 5ac6e3fa..3fc8ecb2 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -195,6 +195,9 @@ impl SysMinion { RequestType::Command => { log::debug!("Master sends a command"); match msg.get_retcode() { + ProtoErrorCode::Success => { + cls.as_ptr().dispatch_command(msg.to_owned()).await; + } ProtoErrorCode::AlreadyConnected => { if MINION_SID.eq(msg.payload()) { log::error!("Another minion from this machine is already connected"); @@ -295,6 +298,83 @@ impl SysMinion { } }); } + + async fn dispatch_command(self: Arc, cmd: MasterMessage) { + log::debug!("Dispatching message"); + let tgt = cmd.get_target(); + + // Is command minion-specific? + if !tgt.id().is_empty() && tgt.id().ne(&self.get_minion_id()) { + log::trace!("Command was dropped as it was specifically addressed for another minion"); + return; + } else if tgt.id().is_empty() { + let traits = get_minion_traits(); + + // Is matching this host? + let mut skip = true; + let hostname = dataconv::as_str(traits.get("system.hostname")); + if !hostname.is_empty() { + for hq in tgt.hostnames() { + if let Ok(hq) = glob::Pattern::new(&hq) { + if hq.matches(&hostname) { + skip = false; + break; + } + } + } + if skip { + log::trace!("Command was dropped as it is specifically targeting different hosts"); + return; + } + } + + // Can match the host, but might not match by traits. + // For example, web* can match "webschool.com" or "webshop.com", + // but traits for those hosts might be different. + let tq = tgt.traits_query(); + if !tq.is_empty() { + match traits::parse_traits_query(tq) { + Ok(q) => { + match traits::to_typed_query(q) { + Ok(tpq) => { + println!("Typed query:\n{:#?}", tpq); + log::debug!("Traits query matches: {:?}", traits::matches_traits(tpq, get_minion_traits())); + } + Err(e) => log::error!("{e}"), + }; + } + Err(e) => log::error!("{e}"), + }; + } + } // else: this minion is directly targeted by its Id. + log::debug!("Dispatched"); + } + + /// Process query + /// Query consists of: + /// + /// 1. Model or state path, starting with the schema (`model://` or `state://` respectively) + /// 2. Traits query (string) + /// + /// The query is parsed on the minion side + fn get_query(q: &str) -> Option<(String, String)> { + // XXX: This is for the minion, not master + let q = q.trim(); + + if !q.starts_with("model://") && !q.starts_with("state://") { + return None; + } + + if !q.contains(" ") { + return Some((q.to_string(), "".to_string())); + } + + let mut tkn = q.splitn(2, |c| c == ' '); + let pth = tkn.next()?.trim(); + let trt = tkn.next()?.trim(); + + Some((pth.to_string(), Regex::new(r"[ \t]+").unwrap().replace_all(trt, " ").to_string())) + } } pub async fn minion(cfp: &str, fingerprint: Option) -> Result<(), SysinspectError> { From 8e981620aa415acd606b405cf8c3a08fa4f9bc4b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:45:55 +0100 Subject: [PATCH 118/148] Remove regex crate as unused --- Cargo.lock | 1 - sysmaster/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6abb42e..5e84b264 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2977,7 +2977,6 @@ dependencies = [ "libsysinspect", "log", "rand", - "regex", "rsa", "rustls", "rustls-pemfile", diff --git a/sysmaster/Cargo.toml b/sysmaster/Cargo.toml index 82a4957c..701b4323 100644 --- a/sysmaster/Cargo.toml +++ b/sysmaster/Cargo.toml @@ -33,4 +33,3 @@ sled = "0.34.7" rsa = { version = "0.9.6", features = ["pkcs5", "sha1", "sha2"] } uuid = { version = "1.11.0", features = ["v4"] } actix-web = "4.9.0" -regex = "1.11.1" From 0fa5d9329004e446f91eaf019fa811c54c6d7fda Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:46:15 +0100 Subject: [PATCH 119/148] Remove commented-out unused code --- sysminion/src/minion.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 3fc8ecb2..3aea4f89 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -136,18 +136,6 @@ impl SysMinion { } } - /* - pub async fn do_qmsg(self: Arc) { - let (_w_chan, mut r_chan) = mpsc::channel(100); - let cls = Arc::new(self.clone()); - tokio::spawn(async move { - while let Some(msg) = r_chan.recv().await { - cls.request(msg).await; - } - }); - } - */ - pub async fn do_proto(self: Arc) -> Result<(), SysinspectError> { let rstm = Arc::clone(&self.rstm); let cls = Arc::new(self.clone()); From 64aaaae9cceb24897ff983b7d48751911eecdeff Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:46:59 +0100 Subject: [PATCH 120/148] Fix references --- sysminion/src/minion.rs | 9 ++++----- sysminion/src/proto.rs | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 3aea4f89..c945c989 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -112,7 +112,7 @@ impl SysMinion { /// Get current minion Id fn get_minion_id(&self) -> String { - dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())) + dataconv::as_str(get_minion_traits().get(traits::SYS_ID)) } /// Talk-back to the master @@ -236,7 +236,7 @@ impl SysMinion { /// Send ehlo pub async fn send_ehlo(self: Arc) -> Result<(), SysinspectError> { let mut r = MinionMessage::new( - dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())), + dataconv::as_str(get_minion_traits().get(traits::SYS_ID)), RequestType::Ehlo, MINION_SID.to_string(), ); @@ -249,8 +249,7 @@ impl SysMinion { /// Send registration request pub async fn send_registration(self: Arc, pbk_pem: String) -> Result<(), SysinspectError> { - let r = - MinionMessage::new(dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())), RequestType::Add, pbk_pem); + let r = MinionMessage::new(dataconv::as_str(get_minion_traits().get(traits::SYS_ID)), RequestType::Add, pbk_pem); log::info!("Registration request to {}", self.cfg.master()); self.request(r.sendable()?).await; @@ -303,7 +302,7 @@ impl SysMinion { let hostname = dataconv::as_str(traits.get("system.hostname")); if !hostname.is_empty() { for hq in tgt.hostnames() { - if let Ok(hq) = glob::Pattern::new(&hq) { + if let Ok(hq) = glob::Pattern::new(hq) { if hq.matches(&hostname) { skip = false; break; diff --git a/sysminion/src/proto.rs b/sysminion/src/proto.rs index 4442ab2c..17d3326d 100644 --- a/sysminion/src/proto.rs +++ b/sysminion/src/proto.rs @@ -13,7 +13,7 @@ pub mod msg { /// Make pong message pub fn get_pong() -> Vec { let p = MinionMessage::new( - dataconv::as_str(get_minion_traits().get(&traits::SYS_ID.to_string())), + dataconv::as_str(get_minion_traits().get(traits::SYS_ID)), RequestType::Pong, MINION_SID.to_string(), ); From 71da409bda385723c6dbf46797d0d6b9fdd0a698 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:47:37 +0100 Subject: [PATCH 121/148] Lintfix --- libsysinspect/src/proto/mod.rs | 69 +++++++++++++++++++++++---------- libsysinspect/src/traits/mod.rs | 2 +- sysmaster/src/master.rs | 3 +- sysminion/src/minion.rs | 4 +- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 9d67d3c9..310b0b22 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -5,14 +5,12 @@ use crate::SysinspectError; use errcodes::ProtoErrorCode; use rqtypes::RequestType; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use serde_json::Value; -use std::collections::HashMap; /// Master message #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MasterMessage { #[serde(rename = "t")] - target: Vec, + target: MinionTarget, #[serde(rename = "r")] request: RequestType, @@ -27,12 +25,12 @@ pub struct MasterMessage { impl MasterMessage { /// Master message constructor pub fn new(rtype: RequestType, data: String) -> MasterMessage { - MasterMessage { target: vec![], request: rtype, data, retcode: ProtoErrorCode::Undef as usize } + MasterMessage { target: Default::default(), request: rtype, data, retcode: ProtoErrorCode::Undef as usize } } /// Add a target. - pub fn add_target(&mut self, t: MinionTarget) { - self.target.push(t); + pub fn set_target(&mut self, t: MinionTarget) { + self.target = t; } /// Set return code @@ -62,6 +60,11 @@ impl MasterMessage { pub fn payload(&self) -> &str { &self.data } + + /// Get targeting means + pub fn get_target(&self) -> &MinionTarget { + &self.target + } } /// Minion message @@ -129,34 +132,58 @@ impl MinionMessage { #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct MinionTarget { /// List of minion Ids - id: Vec, + id: String, - /// List of a collection of traits - #[serde(rename = "t")] - traits: HashMap, + /// Session Id + sid: String, // XXX: Should be gone + + /// Scheme to call (model:// or state://) + #[serde(rename = "s")] + scheme: String, + + /// Traits query that needs to be parsed at Minion + #[serde(rename = "qt")] + traits_query: String, #[serde(rename = "h")] hostnames: Vec, } impl MinionTarget { - pub fn new() -> MinionTarget { - MinionTarget::default() + pub fn new(mid: &str, sid: &str) -> MinionTarget { + MinionTarget { id: mid.to_string(), sid: sid.to_string(), ..Default::default() } } - /// Add target id - pub fn add_minion_id(&mut self, id: String) { - self.id.push(id); + /// Add hostnames + pub fn add_hostname(&mut self, hostname: &str) { + self.hostnames.push(hostname.to_string()); } - /// Add targeting trait - pub fn add_trait(&mut self, tid: String, v: Value) { - self.traits.insert(tid, v); + pub fn id(&self) -> &String { + &self.id } - /// Add hostnames - pub fn add_hostname(&mut self, hostname: &str) { - self.hostnames.push(hostname.to_string()); + pub fn sid(&self) -> &String { + &self.sid + } + + pub fn hostnames(&self) -> &Vec { + &self.hostnames + } + + /// Set scheme + pub fn set_scheme(&mut self, scheme: &str) { + self.scheme = scheme.to_string(); + } + + /// Set traits query + pub fn set_traits_query(&mut self, traits: &str) { + self.traits_query = traits.to_string(); + } + + /// Traits query itself. + pub fn traits_query(&self) -> &String { + &self.traits_query } } diff --git a/libsysinspect/src/traits/mod.rs b/libsysinspect/src/traits/mod.rs index 66a19a1a..123720bf 100644 --- a/libsysinspect/src/traits/mod.rs +++ b/libsysinspect/src/traits/mod.rs @@ -87,7 +87,7 @@ pub fn matches_traits(qt: Vec>>, traits: SystemTraits for ophm in and_op { // op hashmap has always just one key and one value for (opk, opv) in ophm { - and_op_c.push(traits.get(&opk).and_then(|x| Some(x.eq(&opv))).unwrap_or(false)); + and_op_c.push(traits.get(&opk).map(|x| x.eq(&opv)).unwrap_or(false)); } } or_op_c.push(!and_op_c.contains(&false)); diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 00c6a41a..33d6f8d3 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -11,7 +11,6 @@ use libsysinspect::{ proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; -use regex::Regex; use std::{ collections::{HashMap, HashSet}, path::Path, @@ -118,7 +117,7 @@ impl SysMaster { /// Request minion to sync its traits fn msg_request_traits(&mut self, mid: String, sid: String) -> MasterMessage { let mut m = MasterMessage::new(RequestType::Traits, sid.clone()); - let mut tgt = MinionTarget::new(&mid, &sid); + let tgt = MinionTarget::new(&mid, &sid); m.set_target(tgt); m.set_retcode(ProtoErrorCode::Success); diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index c945c989..83b3b143 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -10,9 +10,9 @@ use libsysinspect::{ use once_cell::sync::{Lazy, OnceCell}; use regex::Regex; use std::{fs, path::PathBuf, sync::Arc, vec}; +use tokio::io::AsyncReadExt; use tokio::net::{tcp::OwnedReadHalf, TcpStream}; use tokio::sync::Mutex; -use tokio::{io::AsyncReadExt, sync::mpsc}; use tokio::{io::AsyncWriteExt, net::tcp::OwnedWriteHalf}; use uuid::Uuid; @@ -356,7 +356,7 @@ impl SysMinion { return Some((q.to_string(), "".to_string())); } - let mut tkn = q.splitn(2, |c| c == ' '); + let mut tkn = q.splitn(2, ' '); let pth = tkn.next()?.trim(); let trt = tkn.next()?.trim(); From aa136b58f1e09b18cf678bf84f5df3aba687979c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 15 Nov 2024 22:48:04 +0100 Subject: [PATCH 122/148] Update config with more data. TODO: docs! --- sysinspect.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sysinspect.conf b/sysinspect.conf index 3e4810b2..52b983e3 100644 --- a/sysinspect.conf +++ b/sysinspect.conf @@ -13,5 +13,8 @@ config: # Configuration that is present only on a minion node minion: - master.ip: 192.168.2.102 + # Root directory where minion keeps all data. + # Default: /etc/sysinspect — same as for master + root: /etc/sysinspect + master.ip: 192.168.2.31 master.port: 4200 From 163fd7977df5a5af7331e216a8814b7b29fb3693 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 17 Nov 2024 15:02:48 +0100 Subject: [PATCH 123/148] Run dispatcher in its own spawn --- sysminion/src/minion.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 83b3b143..9888dd98 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -184,7 +184,10 @@ impl SysMinion { log::debug!("Master sends a command"); match msg.get_retcode() { ProtoErrorCode::Success => { - cls.as_ptr().dispatch_command(msg.to_owned()).await; + let cls = cls.as_ptr().clone(); + tokio::spawn(async move { + cls.dispatch_command(msg.to_owned()).await; + }); } ProtoErrorCode::AlreadyConnected => { if MINION_SID.eq(msg.payload()) { @@ -324,8 +327,10 @@ impl SysMinion { Ok(q) => { match traits::to_typed_query(q) { Ok(tpq) => { - println!("Typed query:\n{:#?}", tpq); - log::debug!("Traits query matches: {:?}", traits::matches_traits(tpq, get_minion_traits())); + if !traits::matches_traits(tpq, get_minion_traits()) { + log::trace!("Command was dropped as it does not match the traits"); + return; + } } Err(e) => log::error!("{e}"), }; @@ -334,7 +339,12 @@ impl SysMinion { }; } } // else: this minion is directly targeted by its Id. + + // Valid? + cmd. + log::debug!("Dispatched"); + log::trace!("Command:\n{:#?}", cmd); } /// Process query From 6ad8d10ddd0215f89ade079e5423e3d631e34ecb Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 17 Nov 2024 15:10:49 +0100 Subject: [PATCH 124/148] Use JSON value instead of a plain String for the payload. --- libsysinspect/src/proto/mod.rs | 7 ++++--- sysmaster/src/master.rs | 14 ++++++++------ sysminion/src/minion.rs | 5 ++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 310b0b22..2eb22145 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -5,6 +5,7 @@ use crate::SysinspectError; use errcodes::ProtoErrorCode; use rqtypes::RequestType; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::Value; /// Master message #[derive(Debug, Clone, Serialize, Deserialize)] @@ -16,7 +17,7 @@ pub struct MasterMessage { request: RequestType, #[serde(rename = "d")] - data: String, + data: Value, #[serde(rename = "c")] retcode: usize, @@ -24,7 +25,7 @@ pub struct MasterMessage { impl MasterMessage { /// Master message constructor - pub fn new(rtype: RequestType, data: String) -> MasterMessage { + pub fn new(rtype: RequestType, data: Value) -> MasterMessage { MasterMessage { target: Default::default(), request: rtype, data, retcode: ProtoErrorCode::Undef as usize } } @@ -57,7 +58,7 @@ impl MasterMessage { } /// Get payload - pub fn payload(&self) -> &str { + pub fn payload(&self) -> &Value { &self.data } diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 33d6f8d3..fc92d91a 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -11,6 +11,7 @@ use libsysinspect::{ proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, SysinspectError, }; +use serde_json::json; use std::{ collections::{HashMap, HashSet}, path::Path, @@ -104,7 +105,7 @@ impl SysMaster { tgt.add_hostname(hostname); } - let mut m = MasterMessage::new(RequestType::Command, "SID or something".to_string()); + let mut m = MasterMessage::new(RequestType::Command, json!("SID or something")); m.set_target(tgt); m.set_retcode(ProtoErrorCode::Success); @@ -116,7 +117,7 @@ impl SysMaster { /// Request minion to sync its traits fn msg_request_traits(&mut self, mid: String, sid: String) -> MasterMessage { - let mut m = MasterMessage::new(RequestType::Traits, sid.clone()); + let mut m = MasterMessage::new(RequestType::Traits, json!(sid)); let tgt = MinionTarget::new(&mid, &sid); m.set_target(tgt); m.set_retcode(ProtoErrorCode::Success); @@ -126,7 +127,7 @@ impl SysMaster { /// Already connected fn msg_already_connected(&mut self, mid: String, sid: String) -> MasterMessage { - let mut m = MasterMessage::new(RequestType::Command, sid.clone()); + let mut m = MasterMessage::new(RequestType::Command, json!(sid)); m.set_target(MinionTarget::new(&mid, &sid)); m.set_retcode(ProtoErrorCode::AlreadyConnected); @@ -135,7 +136,8 @@ impl SysMaster { /// Bounce message fn msg_not_registered(&mut self, mid: String) -> MasterMessage { - let mut m = MasterMessage::new(RequestType::AgentUnknown, self.mkr().get_master_key_pem().clone().unwrap().to_string()); + let mut m = + MasterMessage::new(RequestType::AgentUnknown, json!(self.mkr().get_master_key_pem().clone().unwrap().to_string())); m.set_target(MinionTarget::new(&mid, "")); m.set_retcode(ProtoErrorCode::Success); @@ -144,7 +146,7 @@ impl SysMaster { /// Accept registration fn msg_registered(&self, mid: String, msg: &str) -> MasterMessage { - let mut m = MasterMessage::new(RequestType::Reconnect, msg.to_string()); // XXX: Should it be already encrypted? + let mut m = MasterMessage::new(RequestType::Reconnect, json!(msg)); // XXX: Should it be already encrypted? m.set_target(MinionTarget::new(&mid, "")); m.set_retcode(ProtoErrorCode::Success); @@ -313,7 +315,7 @@ impl SysMaster { tokio::spawn(async move { loop { _ = time::sleep(Duration::from_secs(5)).await; - let mut p = MasterMessage::new(RequestType::Ping, "".to_string()); + let mut p = MasterMessage::new(RequestType::Ping, json!("")); let mut t = MinionTarget::default(); t.add_hostname("*"); p.set_target(t); diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 9888dd98..227b41dc 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -207,8 +207,8 @@ impl SysMinion { } } RequestType::AgentUnknown => { - let pbk_pem = msg.payload(); // Expected PEM RSA pub key - let (_, pbk) = rsa::keys::from_pem(None, Some(pbk_pem)).unwrap(); + let pbk_pem = dataconv::as_str(Some(msg.payload()).cloned()); // Expected PEM RSA pub key + let (_, pbk) = rsa::keys::from_pem(None, Some(&pbk_pem)).unwrap(); let fpt = rsa::keys::get_fingerprint(&pbk.unwrap()).unwrap(); log::error!("Minion is not registered"); @@ -341,7 +341,6 @@ impl SysMinion { } // else: this minion is directly targeted by its Id. // Valid? - cmd. log::debug!("Dispatched"); log::trace!("Command:\n{:#?}", cmd); From 273d6ccf9ea0f9ea405754826bf52f95e7995cf2 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sun, 17 Nov 2024 15:45:08 +0100 Subject: [PATCH 125/148] Add readmes to help understanding that each package is for --- libsysinspect/src/intp/README.txt | 1 + libsysinspect/src/mdescr/README.txt | 1 + libsysinspect/src/modlib/README.txt | 1 + libsysinspect/src/reactor/README.txt | 2 ++ libsysinspect/src/traits/README.txt | 1 + libsysinspect/src/util/README.txt | 1 + 6 files changed, 7 insertions(+) create mode 100644 libsysinspect/src/intp/README.txt create mode 100644 libsysinspect/src/mdescr/README.txt create mode 100644 libsysinspect/src/modlib/README.txt create mode 100644 libsysinspect/src/reactor/README.txt create mode 100644 libsysinspect/src/traits/README.txt create mode 100644 libsysinspect/src/util/README.txt diff --git a/libsysinspect/src/intp/README.txt b/libsysinspect/src/intp/README.txt new file mode 100644 index 00000000..cf08bd8c --- /dev/null +++ b/libsysinspect/src/intp/README.txt @@ -0,0 +1 @@ +Package interpreting module spec (runtime) \ No newline at end of file diff --git a/libsysinspect/src/mdescr/README.txt b/libsysinspect/src/mdescr/README.txt new file mode 100644 index 00000000..2d79db6f --- /dev/null +++ b/libsysinspect/src/mdescr/README.txt @@ -0,0 +1 @@ +Package to work with the model spec (before actual runtime) \ No newline at end of file diff --git a/libsysinspect/src/modlib/README.txt b/libsysinspect/src/modlib/README.txt new file mode 100644 index 00000000..fc766f85 --- /dev/null +++ b/libsysinspect/src/modlib/README.txt @@ -0,0 +1 @@ +Package to work with the external inspection modules. \ No newline at end of file diff --git a/libsysinspect/src/reactor/README.txt b/libsysinspect/src/reactor/README.txt new file mode 100644 index 00000000..b82bbdc9 --- /dev/null +++ b/libsysinspect/src/reactor/README.txt @@ -0,0 +1,2 @@ +Package that implements reactor, formatters, handlers etc. +Everything related to them should go here. \ No newline at end of file diff --git a/libsysinspect/src/traits/README.txt b/libsysinspect/src/traits/README.txt new file mode 100644 index 00000000..b014612c --- /dev/null +++ b/libsysinspect/src/traits/README.txt @@ -0,0 +1 @@ +System traits package. \ No newline at end of file diff --git a/libsysinspect/src/util/README.txt b/libsysinspect/src/util/README.txt new file mode 100644 index 00000000..35646e63 --- /dev/null +++ b/libsysinspect/src/util/README.txt @@ -0,0 +1 @@ +Various utilities: data conversion, filesystem reads etc. \ No newline at end of file From e75228abc52aa562d8a695b41a655b6d6e993fb2 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:19:03 +0100 Subject: [PATCH 126/148] Update dependencies --- Cargo.lock | 1 + libsysinspect/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 5e84b264..a4ed1e64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1675,6 +1675,7 @@ dependencies = [ "base64", "chrono", "colored", + "hex", "indexmap", "lazy_static", "log", diff --git a/libsysinspect/Cargo.toml b/libsysinspect/Cargo.toml index 6baed770..8185bdf8 100644 --- a/libsysinspect/Cargo.toml +++ b/libsysinspect/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" base64 = "0.22.1" chrono = "0.4.38" colored = "2.1.0" +hex = "0.4.3" indexmap = { version = "2.6.0", features = ["serde"] } lazy_static = "1.5.0" log = "0.4.22" From a012d8feb1cb4e1011490f30d3c0cac0430ed8af Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:21:02 +0100 Subject: [PATCH 127/148] Add model/state source autosync. It is using SHA256 checksum to download only what changed. --- libsysinspect/src/cfg/mmconf.rs | 37 ++++++-- libsysinspect/src/lib.rs | 2 +- libsysinspect/src/mdescr/distr.rs | 35 ++++++++ libsysinspect/src/mdescr/mod.rs | 1 + libsysinspect/src/proto/mod.rs | 6 ++ libsysinspect/src/proto/payload.rs | 88 ++++++++++++++++++ libsysinspect/src/util/iofs.rs | 54 +++++++++++ libsysinspect/src/util/mod.rs | 1 + sysmaster/src/master.rs | 34 +++++-- sysminion/src/filedata.rs | 51 +++++++++++ sysminion/src/main.rs | 1 + sysminion/src/minion.rs | 140 ++++++++++++++++++++--------- 12 files changed, 394 insertions(+), 56 deletions(-) create mode 100644 libsysinspect/src/mdescr/distr.rs create mode 100644 libsysinspect/src/proto/payload.rs create mode 100644 libsysinspect/src/util/iofs.rs create mode 100644 sysminion/src/filedata.rs diff --git a/libsysinspect/src/cfg/mmconf.rs b/libsysinspect/src/cfg/mmconf.rs index 93ee7036..6ac49a38 100644 --- a/libsysinspect/src/cfg/mmconf.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -104,10 +104,18 @@ pub struct MasterConfig { socket: Option, #[serde(rename = "fileserver.bind.ip")] - fileserver_ip: Option, + fsr_ip: Option, #[serde(rename = "fileserver.bind.port")] - fileserver_port: Option, + fsr_port: Option, + + // Exported models path root on the fileserver + #[serde(rename = "fileserver.models.root")] + fsr_models_root: String, + + // Exported models on the fileserver + #[serde(rename = "fileserver.models")] + fsr_models: Vec, } impl MasterConfig { @@ -138,14 +146,14 @@ impl MasterConfig { pub fn fileserver_bind_addr(&self) -> String { format!( "{}:{}", - self.fileserver_ip.to_owned().unwrap_or(DEFAULT_ADDR.to_string()), - self.fileserver_port.unwrap_or(DEFAULT_FILESERVER_PORT) + self.fsr_ip.to_owned().unwrap_or(DEFAULT_ADDR.to_string()), + self.fsr_port.unwrap_or(DEFAULT_FILESERVER_PORT) ) } - /// Get default sysinspect root. For master it is always /etc/sysinspect - pub fn root_dir(&self) -> PathBuf { - PathBuf::from(DEFAULT_SYSINSPECT_ROOT.to_string()) + /// Get a list of exported models from the fileserver + pub fn fileserver_models(&self) -> &Vec { + &self.fsr_models } /// Get fileserver root @@ -153,6 +161,21 @@ impl MasterConfig { self.root_dir().join(CFG_FILESERVER_ROOT) } + /// Get models root on the fileserver + pub fn fileserver_mdl_root(&self, alone: bool) -> PathBuf { + let mr = PathBuf::from(&self.fsr_models_root.strip_prefix("/").unwrap_or_default()); + if alone { + mr + } else { + self.fileserver_root().join(mr) + } + } + + /// Get default sysinspect root. For master it is always /etc/sysinspect + pub fn root_dir(&self) -> PathBuf { + PathBuf::from(DEFAULT_SYSINSPECT_ROOT.to_string()) + } + /// Get minion keys store pub fn keyman_root(&self) -> PathBuf { self.root_dir().join(CFG_MINION_KEYS) diff --git a/libsysinspect/src/lib.rs b/libsysinspect/src/lib.rs index 6f03613a..80aa2476 100644 --- a/libsysinspect/src/lib.rs +++ b/libsysinspect/src/lib.rs @@ -110,6 +110,6 @@ impl From> for SysinspectError { impl From> for SysinspectError { fn from(err: Box) -> SysinspectError { - SysinspectError::AsynDynError(err) + SysinspectError::DynError(err) } } diff --git a/libsysinspect/src/mdescr/distr.rs b/libsysinspect/src/mdescr/distr.rs new file mode 100644 index 00000000..ad14409c --- /dev/null +++ b/libsysinspect/src/mdescr/distr.rs @@ -0,0 +1,35 @@ +/* +Distributed Model Spec. + +This package contains utilities to distribute and setup a published +model with all its derivatives across all the minions, so they can +pick it up and process. +*/ + +use std::{collections::HashMap, path::PathBuf}; +use walkdir::WalkDir; + +use crate::util::iofs::get_file_sha256; + +use super::mspec::MODEL_FILE_EXT; + +/// Gets modes files from the `pth` as root. +/// Returns `Vec` of relative paths to the `pth`. +/// Used in for the fileserver, so the minion knows +/// the paths to download. +pub fn model_files(pth: PathBuf) -> HashMap { + let ext = MODEL_FILE_EXT.strip_prefix(".").unwrap_or_default(); + WalkDir::new(&pth) + .into_iter() + .filter_map(|entry| { + let entry = entry.ok()?; + + if entry.file_type().is_file() && entry.path().extension().and_then(|e| e.to_str()) == Some(ext) { + let relative_path = entry.path().strip_prefix(&pth).ok()?.to_string_lossy().to_string(); + Some((relative_path, get_file_sha256(entry.path().to_path_buf()).unwrap_or_default())) + } else { + None + } + }) + .collect::>() +} diff --git a/libsysinspect/src/mdescr/mod.rs b/libsysinspect/src/mdescr/mod.rs index aad626f8..5661d10d 100644 --- a/libsysinspect/src/mdescr/mod.rs +++ b/libsysinspect/src/mdescr/mod.rs @@ -1,4 +1,5 @@ pub mod datapatch; +pub mod distr; pub mod mspec; pub mod mspecdef; diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index 2eb22145..e0e361fb 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -1,4 +1,5 @@ pub mod errcodes; +pub mod payload; pub mod rqtypes; use crate::SysinspectError; @@ -172,6 +173,11 @@ impl MinionTarget { &self.hostnames } + /// Get scheme + pub fn scheme(&self) -> &String { + &self.scheme + } + /// Set scheme pub fn set_scheme(&mut self, scheme: &str) { self.scheme = scheme.to_string(); diff --git a/libsysinspect/src/proto/payload.rs b/libsysinspect/src/proto/payload.rs new file mode 100644 index 00000000..51f4b1bb --- /dev/null +++ b/libsysinspect/src/proto/payload.rs @@ -0,0 +1,88 @@ +// XXX: Refactor: move all message types-related code here! + +/* +Payload types and their deserialisation. +*/ + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{from_value, Value}; + +/// Payload types +pub enum PayloadType { + ModelOrStatement(ModStatePayload), + Undef(Value), +} + +impl TryFrom for PayloadType { + type Error = serde_json::Error; + + fn try_from(value: Value) -> Result { + if let Ok(v) = from_value::(value.clone()) { + return Ok(PayloadType::ModelOrStatement(v)); + } + Ok(PayloadType::Undef(value)) + } +} + +/// Message is sent to the Minion +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ModStatePayload { + // Each file has a SHA1 checksum to prevent huge bogus traffic + files: HashMap, + + // Root where models starts. It corresponds to "fileserver.models.root" conf of Master. + // It will be substracted from each file path when saving + models_root: String, + + // session Id + sid: String, + + // sysinspect URI + uri: String, +} + +impl ModStatePayload { + pub fn new(sid: String) -> Self { + ModStatePayload { sid, ..Default::default() } + } + + /// Set URI + pub fn set_uri(mut self, uri: String) -> Self { + self.uri = uri; + self + } + + /// Add files + pub fn add_files(mut self, files: HashMap) -> Self { + self.files.extend(files); + self + } + + /// Set models root + pub fn set_models_root(mut self, mr: &str) -> Self { + self.models_root = mr.to_string(); + self + } + + /// Get list of files to download + pub fn files(&self) -> &HashMap { + &self.files + } + + /// Get SID + pub fn sid(&self) -> &str { + &self.sid + } + + /// Get URI + pub fn uri(&self) -> &str { + &self.uri + } + + /// Get root of models + pub fn models_root(&self) -> &str { + &self.models_root + } +} diff --git a/libsysinspect/src/util/iofs.rs b/libsysinspect/src/util/iofs.rs new file mode 100644 index 00000000..18ad7d41 --- /dev/null +++ b/libsysinspect/src/util/iofs.rs @@ -0,0 +1,54 @@ +/* +Various unsorted utils with the filesystem, IO, files, byte arrays etc +*/ + +use crate::SysinspectError; +use hex::encode; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + io::{BufReader, Read}, +}; +use std::{fs::File, path::PathBuf}; +use walkdir::WalkDir; + +/// Calculate an SHA265 checksum of a file on the file system. +pub fn get_file_sha256(pth: PathBuf) -> Result { + let mut dg = Sha256::new(); + let mut buf = [0u8; 0x2000]; + let mut reader = BufReader::new(File::open(&pth)?); + loop { + let brd = reader.read(&mut buf)?; + if brd == 0 { + break; + } + dg.update(&buf[..brd]); + } + + Ok(encode(dg.finalize())) +} + +/// Scan a given root for any file. +/// Returns a `HashMap` with format `path` to `checksum`. +pub fn scan_files_sha256(pth: PathBuf, ext: Option<&str>) -> HashMap { + let ext = ext.and_then(|e| Some(e.trim_start_matches('.'))); // Just in case :) + WalkDir::new(&pth) + .into_iter() + .filter_map(|entry| { + let entry = entry.ok()?; + + if entry.file_type().is_file() { + if ext.is_some() && entry.path().extension().and_then(|e| e.to_str()) == ext || ext.is_none() { + Some(( + entry.path().strip_prefix(&pth).ok()?.to_string_lossy().to_string(), + get_file_sha256(entry.path().to_path_buf()).unwrap_or_default(), + )) + } else { + None + } + } else { + None + } + }) + .collect::>() +} diff --git a/libsysinspect/src/util/mod.rs b/libsysinspect/src/util/mod.rs index 77c2b205..03fef808 100644 --- a/libsysinspect/src/util/mod.rs +++ b/libsysinspect/src/util/mod.rs @@ -1 +1,2 @@ pub mod dataconv; +pub mod iofs; diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index fc92d91a..0ae6e0be 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -8,9 +8,14 @@ use crate::{ }; use libsysinspect::{ cfg::mmconf::MasterConfig, - proto::{self, errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion}, + mdescr::distr::model_files, + proto::{ + self, errcodes::ProtoErrorCode, payload::ModStatePayload, rqtypes::RequestType, MasterMessage, MinionMessage, + MinionTarget, ProtoConversion, + }, SysinspectError, }; +use rustls::crypto::hash::Hash; use serde_json::json; use std::{ collections::{HashMap, HashSet}, @@ -94,6 +99,7 @@ impl SysMaster { &mut self.mkr } + /// Construct a Command message to the minion fn msg_query(&mut self, payload: &str) -> Option { let query = payload.split(";").map(|s| s.to_string()).collect::>(); @@ -105,11 +111,28 @@ impl SysMaster { tgt.add_hostname(hostname); } - let mut m = MasterMessage::new(RequestType::Command, json!("SID or something")); - m.set_target(tgt); - m.set_retcode(ProtoErrorCode::Success); + // Collect downloadable model(s) files + let mut out: HashMap = HashMap::default(); + for em in self.cfg.fileserver_models() { + for (n, cs) in model_files(self.cfg.fileserver_mdl_root(false).join(&em)) { + out.insert( + format!("/{}/{em}/{n}", self.cfg.fileserver_mdl_root(false).file_name().unwrap().to_str().unwrap()), + cs, + ); + } + } - return Some(m); + let mut msg = MasterMessage::new( + RequestType::Command, + json!(ModStatePayload::new("12345".to_string()) + .set_uri(scheme.to_string()) + .add_files(out) + .set_models_root(self.cfg.fileserver_mdl_root(true).to_str().unwrap_or_default())), // TODO: SID part + ); + msg.set_target(tgt); + msg.set_retcode(ProtoErrorCode::Success); + + return Some(msg); } None @@ -286,7 +309,6 @@ impl SysMaster { Ok(Some(payload)) => { log::info!("Querying minions: {}", payload); if let Some(msg) = master.lock().await.msg_query(&payload) { - log::debug!("{:#?}", msg); let _ = bcast.send(msg.sendable().unwrap()); } } diff --git a/sysminion/src/filedata.rs b/sysminion/src/filedata.rs new file mode 100644 index 00000000..13f4aff3 --- /dev/null +++ b/sysminion/src/filedata.rs @@ -0,0 +1,51 @@ +/* +Filedata manager + */ + +use libsysinspect::{mdescr::mspec::MODEL_FILE_EXT, util::iofs::scan_files_sha256, SysinspectError}; +use std::{collections::HashMap, path::PathBuf}; + +#[derive(Debug, Default, Clone)] +pub struct MinionFiledata { + // Checksum to file + stack: HashMap, + + // Path to models (root) + mpth: PathBuf, +} + +impl MinionFiledata { + /// Constructor + pub fn new(mpth: PathBuf) -> Result { + let mut instance = Self { mpth, ..Default::default() }; + instance.init(); + + Ok(instance) + } + + pub fn init(&mut self) { + self.stack = scan_files_sha256(self.mpth.to_owned(), Some(MODEL_FILE_EXT)) + .iter() + .map(|(f, cs)| (cs.to_owned(), PathBuf::from(f.to_owned()))) + .collect::>(); + } + + /// Verify if a corresponding file matches the checksum + pub fn check_sha256(&self, pth: String, cs: String, relative: bool) -> bool { + if !self.stack.contains_key(&cs) { + // Unknown checksum, (re)download required + return false; + } + + if let Some(p) = self.stack.get(&cs) { + let pth = PathBuf::from(pth); + if relative { + return pth.ends_with(p); + } else { + return pth.eq(p); + } + } + + false + } +} diff --git a/sysminion/src/main.rs b/sysminion/src/main.rs index 0a12291e..a497e761 100644 --- a/sysminion/src/main.rs +++ b/sysminion/src/main.rs @@ -1,4 +1,5 @@ mod clidef; +mod filedata; mod minion; mod proto; mod rsa; diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 227b41dc..4504f7cc 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,14 +1,18 @@ -use crate::{proto, rsa::MinionRSAKeyManager}; +use crate::{filedata::MinionFiledata, proto, rsa::MinionRSAKeyManager}; use libsysinspect::{ cfg::{self, mmconf::MinionConfig}, - proto::{errcodes::ProtoErrorCode, rqtypes::RequestType, MasterMessage, MinionMessage, ProtoConversion}, + proto::{ + errcodes::ProtoErrorCode, + payload::{ModStatePayload, PayloadType}, + rqtypes::RequestType, + MasterMessage, MinionMessage, ProtoConversion, + }, rsa, traits::{self, systraits::SystemTraits}, util::dataconv, SysinspectError, }; use once_cell::sync::{Lazy, OnceCell}; -use regex::Regex; use std::{fs, path::PathBuf, sync::Arc, vec}; use tokio::io::AsyncReadExt; use tokio::net::{tcp::OwnedReadHalf, TcpStream}; @@ -39,6 +43,8 @@ pub struct SysMinion { rstm: Arc>, wstm: Arc>, + + filedata: Mutex, } impl SysMinion { @@ -60,6 +66,7 @@ impl SysMinion { kman: MinionRSAKeyManager::new(cfg.root_dir())?, rstm: Arc::new(Mutex::new(rstm)), wstm: Arc::new(Mutex::new(wstm)), + filedata: Mutex::new(MinionFiledata::new(cfg.models_dir())?), }; instance.init()?; @@ -164,7 +171,7 @@ impl SysMinion { } }; - log::trace!("Received: {:?}", msg); + log::trace!("Received: {:#?}", msg); match msg.req_type() { RequestType::Add => { @@ -186,7 +193,7 @@ impl SysMinion { ProtoErrorCode::Success => { let cls = cls.as_ptr().clone(); tokio::spawn(async move { - cls.dispatch_command(msg.to_owned()).await; + cls.dispatch(msg.to_owned()).await; }); } ProtoErrorCode::AlreadyConnected => { @@ -260,12 +267,13 @@ impl SysMinion { } /// Download a file from master - async fn download_file(self: Arc, fname: String) { + async fn download_file(self: Arc, fname: &str) -> Result, SysinspectError> { async fn fetch_file(url: &str, filename: &str) -> Result { - let rsp = match reqwest::get(format!("http://{}/{}", url, filename)).await { + let url = format!("http://{}/{}", url.trim_end_matches('/'), filename.to_string().trim_start_matches('/')); + let rsp = match reqwest::get(url.to_owned()).await { Ok(rsp) => rsp, Err(err) => { - return Err(SysinspectError::MinionGeneralError(format!("{}", err))); + return Err(SysinspectError::MinionGeneralError(format!("Unable to get file at {url}: {err}"))); } }; @@ -273,7 +281,7 @@ impl SysMinion { reqwest::StatusCode::OK => match rsp.text().await { Ok(data) => data, Err(err) => { - return Err(SysinspectError::MinionGeneralError(format!("{}", err))); + return Err(SysinspectError::MinionGeneralError(format!("Unable to get text from the file: {}", err))); } }, reqwest::StatusCode::NOT_FOUND => return Err(SysinspectError::MinionGeneralError("File not found".to_string())), @@ -281,15 +289,80 @@ impl SysMinion { }) } let addr = self.cfg.fileserver(); - tokio::spawn(async move { + let fname = fname.to_string(); + let h = tokio::spawn(async move { match fetch_file(&addr, &fname).await { - Ok(data) => log::debug!("Result returned as {:#?}", data), - Err(err) => log::error!("{err}"), + Ok(data) => { + log::debug!("Filename: {fname} contains {} bytes", data.len()); + Some(data.into_bytes()) + } + Err(err) => { + log::error!("Error while downloading file {fname}: {err}"); + None + } } }); + + if let Ok(Some(data)) = h.await { + return Ok(data); + } + + Err(SysinspectError::MinionGeneralError("File was not downloaded".to_string())) } - async fn dispatch_command(self: Arc, cmd: MasterMessage) { + /// Launch sysinspect + async fn launch_sysinspect(self: Arc, msp: &ModStatePayload) { + // TODO: Now dispatch sysinspect! + // + // 1. [ ] Check if files are there, if not download them + // 2. [ ] Render the DSL according to the traits + // 3. [ ] Run the model + // 4. [ ] Collect the output and send back + + // Check if all files are there. + let cls = Arc::new(self); + let mut dirty = false; + for (uri_file, fcs) in msp.files() { + let dst = cls + .cfg + .models_dir() + .join(uri_file.trim_start_matches(&format!("/{}", msp.models_root())).strip_prefix("/").unwrap_or_default()); + + if cls.as_ptr().filedata.lock().await.check_sha256(uri_file.to_owned(), fcs.to_owned(), true) { + continue; + } + log::debug!("File {uri_file} has different checksum"); + + match cls.as_ptr().download_file(uri_file).await { + Ok(data) => { + let dst_dir = dst.parent().unwrap(); + if !dst_dir.exists() { + log::debug!("Creating directory: {:?}", dst_dir); + if let Err(err) = fs::create_dir_all(dst_dir) { + log::error!("Unable to create directories for model download: {err}"); + return; + } + } + + log::debug!("Saving URI {uri_file} as {:?}", dst_dir); + if let Err(err) = fs::write(dst.to_owned(), data) { + log::error!("Unable to save downloaded file to {:?}: {err}", dst); + return; + } + dirty = true; + } + Err(_) => todo!(), + } + } + + if dirty { + cls.as_ptr().filedata.lock().await.init(); + } + + log::warn!("Launching sysinspect. Does it work?"); + } + + async fn dispatch(self: Arc, cmd: MasterMessage) { log::debug!("Dispatching message"); let tgt = cmd.get_target(); @@ -340,36 +413,19 @@ impl SysMinion { } } // else: this minion is directly targeted by its Id. - // Valid? - - log::debug!("Dispatched"); - log::trace!("Command:\n{:#?}", cmd); - } - - /// Process query - /// Query consists of: - /// - /// 1. Model or state path, starting with the schema (`model://` or `state://` respectively) - /// 2. Traits query (string) - /// - /// The query is parsed on the minion side - fn get_query(q: &str) -> Option<(String, String)> { - // XXX: This is for the minion, not master - let q = q.trim(); - - if !q.starts_with("model://") && !q.starts_with("state://") { - return None; - } - - if !q.contains(" ") { - return Some((q.to_string(), "".to_string())); + match PayloadType::try_from(cmd.payload().clone()) { + Ok(PayloadType::ModelOrStatement(pld)) => { + self.launch_sysinspect(&pld).await; + log::debug!("Command dispatched"); + log::trace!("Command payload: {:#?}", pld); + } + Ok(PayloadType::Undef(pld)) => { + log::error!("Unknown command: {:#?}", pld); + } + Err(err) => { + log::error!("Error dispatching command: {err}"); + } } - - let mut tkn = q.splitn(2, ' '); - let pth = tkn.next()?.trim(); - let trt = tkn.next()?.trim(); - - Some((pth.to_string(), Regex::new(r"[ \t]+").unwrap().replace_all(trt, " ").to_string())) } } From e18d4c04456f8a05e708d21d37e74da289c659d1 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:21:41 +0100 Subject: [PATCH 128/148] Update config file --- sysinspect.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sysinspect.conf b/sysinspect.conf index 52b983e3..26925596 100644 --- a/sysinspect.conf +++ b/sysinspect.conf @@ -11,6 +11,14 @@ config: fileserver.bind.ip: 0.0.0.0 fileserver.bind.port: 4201 + # Path to the models in fileserver root + fileserver.models.root: /models + + # Exported models + fileserver.models: + - router + - inherited + # Configuration that is present only on a minion node minion: # Root directory where minion keeps all data. From 18e2ffcf698bacb6c74015b5bfca432920f364df Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:21:53 +0100 Subject: [PATCH 129/148] remove done note --- libsysinspect/src/mdescr/mspec.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/libsysinspect/src/mdescr/mspec.rs b/libsysinspect/src/mdescr/mspec.rs index 758a10d9..755ed7a2 100644 --- a/libsysinspect/src/mdescr/mspec.rs +++ b/libsysinspect/src/mdescr/mspec.rs @@ -66,7 +66,6 @@ impl SpecLoader { fn merge_parts(&mut self, chunks: &mut Vec) -> Result { if chunks.is_empty() { return Err(SysinspectError::ModelMultipleIndex("No data found".to_string())); - // XXX: Add one more exception } let mut base = chunks.remove(0); From de1844d9db8eaaf1a0e1825f7986373090a05c17 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:24:02 +0100 Subject: [PATCH 130/148] Update internal comments, remove forgotten warn!() as should be debug!() --- sysminion/src/minion.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 4504f7cc..71480afa 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -314,7 +314,7 @@ impl SysMinion { async fn launch_sysinspect(self: Arc, msp: &ModStatePayload) { // TODO: Now dispatch sysinspect! // - // 1. [ ] Check if files are there, if not download them + // 1. [x] Check if files are there, if not download them // 2. [ ] Render the DSL according to the traits // 3. [ ] Run the model // 4. [ ] Collect the output and send back @@ -359,7 +359,8 @@ impl SysMinion { cls.as_ptr().filedata.lock().await.init(); } - log::warn!("Launching sysinspect. Does it work?"); + log::debug!("Launching model for sysinspect"); + // TODO: launch sysinspect here } async fn dispatch(self: Arc, cmd: MasterMessage) { From d9d1a9010f4d969e9c63939026e1d887703a895c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:25:58 +0100 Subject: [PATCH 131/148] Simplify trims on option --- libsysinspect/src/util/iofs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsysinspect/src/util/iofs.rs b/libsysinspect/src/util/iofs.rs index 18ad7d41..cde37c79 100644 --- a/libsysinspect/src/util/iofs.rs +++ b/libsysinspect/src/util/iofs.rs @@ -31,7 +31,7 @@ pub fn get_file_sha256(pth: PathBuf) -> Result { /// Scan a given root for any file. /// Returns a `HashMap` with format `path` to `checksum`. pub fn scan_files_sha256(pth: PathBuf, ext: Option<&str>) -> HashMap { - let ext = ext.and_then(|e| Some(e.trim_start_matches('.'))); // Just in case :) + let ext = ext.map(|e| e.trim_start_matches('.')); // Just in case :) WalkDir::new(&pth) .into_iter() .filter_map(|entry| { From 740c202114e35735455e98b8e5b7bfe8986f994e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:26:10 +0100 Subject: [PATCH 132/148] Consume to cleanup the memory --- sysmaster/src/master.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 0ae6e0be..5bddc554 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -15,7 +15,6 @@ use libsysinspect::{ }, SysinspectError, }; -use rustls::crypto::hash::Hash; use serde_json::json; use std::{ collections::{HashMap, HashSet}, @@ -114,7 +113,7 @@ impl SysMaster { // Collect downloadable model(s) files let mut out: HashMap = HashMap::default(); for em in self.cfg.fileserver_models() { - for (n, cs) in model_files(self.cfg.fileserver_mdl_root(false).join(&em)) { + for (n, cs) in model_files(self.cfg.fileserver_mdl_root(false).join(em)) { out.insert( format!("/{}/{em}/{n}", self.cfg.fileserver_mdl_root(false).file_name().unwrap().to_str().unwrap()), cs, From b85a95fa81f97e633ef1cb0f285de12d82d3831e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:26:29 +0100 Subject: [PATCH 133/148] Prevent unnecessary mem consumption --- sysminion/src/minion.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 71480afa..7daf8df4 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -345,7 +345,7 @@ impl SysMinion { } log::debug!("Saving URI {uri_file} as {:?}", dst_dir); - if let Err(err) = fs::write(dst.to_owned(), data) { + if let Err(err) = fs::write(&dst, data) { log::error!("Unable to save downloaded file to {:?}: {err}", dst); return; } From 3a15974276b810bff245a8420b9ddb60d84b49b8 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 16:58:07 +0100 Subject: [PATCH 134/148] Remove legacy code --- libsysinspect/src/mdescr/distr.rs | 35 ------------------------------- libsysinspect/src/mdescr/mod.rs | 1 - sysmaster/src/master.rs | 5 +++-- 3 files changed, 3 insertions(+), 38 deletions(-) delete mode 100644 libsysinspect/src/mdescr/distr.rs diff --git a/libsysinspect/src/mdescr/distr.rs b/libsysinspect/src/mdescr/distr.rs deleted file mode 100644 index ad14409c..00000000 --- a/libsysinspect/src/mdescr/distr.rs +++ /dev/null @@ -1,35 +0,0 @@ -/* -Distributed Model Spec. - -This package contains utilities to distribute and setup a published -model with all its derivatives across all the minions, so they can -pick it up and process. -*/ - -use std::{collections::HashMap, path::PathBuf}; -use walkdir::WalkDir; - -use crate::util::iofs::get_file_sha256; - -use super::mspec::MODEL_FILE_EXT; - -/// Gets modes files from the `pth` as root. -/// Returns `Vec` of relative paths to the `pth`. -/// Used in for the fileserver, so the minion knows -/// the paths to download. -pub fn model_files(pth: PathBuf) -> HashMap { - let ext = MODEL_FILE_EXT.strip_prefix(".").unwrap_or_default(); - WalkDir::new(&pth) - .into_iter() - .filter_map(|entry| { - let entry = entry.ok()?; - - if entry.file_type().is_file() && entry.path().extension().and_then(|e| e.to_str()) == Some(ext) { - let relative_path = entry.path().strip_prefix(&pth).ok()?.to_string_lossy().to_string(); - Some((relative_path, get_file_sha256(entry.path().to_path_buf()).unwrap_or_default())) - } else { - None - } - }) - .collect::>() -} diff --git a/libsysinspect/src/mdescr/mod.rs b/libsysinspect/src/mdescr/mod.rs index 5661d10d..aad626f8 100644 --- a/libsysinspect/src/mdescr/mod.rs +++ b/libsysinspect/src/mdescr/mod.rs @@ -1,5 +1,4 @@ pub mod datapatch; -pub mod distr; pub mod mspec; pub mod mspecdef; diff --git a/sysmaster/src/master.rs b/sysmaster/src/master.rs index 5bddc554..b83aef61 100644 --- a/sysmaster/src/master.rs +++ b/sysmaster/src/master.rs @@ -8,11 +8,12 @@ use crate::{ }; use libsysinspect::{ cfg::mmconf::MasterConfig, - mdescr::distr::model_files, + mdescr::mspec::MODEL_FILE_EXT, proto::{ self, errcodes::ProtoErrorCode, payload::ModStatePayload, rqtypes::RequestType, MasterMessage, MinionMessage, MinionTarget, ProtoConversion, }, + util::iofs::scan_files_sha256, SysinspectError, }; use serde_json::json; @@ -113,7 +114,7 @@ impl SysMaster { // Collect downloadable model(s) files let mut out: HashMap = HashMap::default(); for em in self.cfg.fileserver_models() { - for (n, cs) in model_files(self.cfg.fileserver_mdl_root(false).join(em)) { + for (n, cs) in scan_files_sha256(self.cfg.fileserver_mdl_root(false).join(em), Some(MODEL_FILE_EXT)) { out.insert( format!("/{}/{em}/{n}", self.cfg.fileserver_mdl_root(false).file_name().unwrap().to_str().unwrap()), cs, From dd8056fa954c5708c14fe7a9e2cf78db3ef46e3e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 17:14:31 +0100 Subject: [PATCH 135/148] Pass query scheme to the sysinspect launcher --- sysminion/src/minion.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 7daf8df4..0aa802c8 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -311,7 +311,7 @@ impl SysMinion { } /// Launch sysinspect - async fn launch_sysinspect(self: Arc, msp: &ModStatePayload) { + async fn launch_sysinspect(self: Arc, scheme: &str, msp: &ModStatePayload) { // TODO: Now dispatch sysinspect! // // 1. [x] Check if files are there, if not download them @@ -319,7 +319,7 @@ impl SysMinion { // 3. [ ] Run the model // 4. [ ] Collect the output and send back - // Check if all files are there. + // Auto-sync all data files let cls = Arc::new(self); let mut dirty = false; for (uri_file, fcs) in msp.files() { @@ -359,7 +359,7 @@ impl SysMinion { cls.as_ptr().filedata.lock().await.init(); } - log::debug!("Launching model for sysinspect"); + log::debug!("Launching model for sysinspect for: {scheme}"); // TODO: launch sysinspect here } @@ -416,7 +416,7 @@ impl SysMinion { match PayloadType::try_from(cmd.payload().clone()) { Ok(PayloadType::ModelOrStatement(pld)) => { - self.launch_sysinspect(&pld).await; + self.launch_sysinspect(cmd.get_target().scheme(), &pld).await; log::debug!("Command dispatched"); log::trace!("Command payload: {:#?}", pld); } From a83530b8f0302c757f36c5f76b09dcb9b281a4fc Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 18:43:18 +0100 Subject: [PATCH 136/148] Add sysinspect model query parser --- libsysinspect/src/proto/mod.rs | 1 + libsysinspect/src/proto/query.rs | 79 ++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 libsysinspect/src/proto/query.rs diff --git a/libsysinspect/src/proto/mod.rs b/libsysinspect/src/proto/mod.rs index e0e361fb..de9d9fc1 100644 --- a/libsysinspect/src/proto/mod.rs +++ b/libsysinspect/src/proto/mod.rs @@ -1,5 +1,6 @@ pub mod errcodes; pub mod payload; +pub mod query; pub mod rqtypes; use crate::SysinspectError; diff --git a/libsysinspect/src/proto/query.rs b/libsysinspect/src/proto/query.rs new file mode 100644 index 00000000..b21cca57 --- /dev/null +++ b/libsysinspect/src/proto/query.rs @@ -0,0 +1,79 @@ +use std::sync::{Arc, Mutex}; + +use crate::SysinspectError; + +/// Targeting schemes +pub static SCHEME_MODEL: &str = "model://"; +pub static SCHEME_STATE: &str = "state://"; + +/// +/// Query parser (scheme). +/// It has the following format: +/// +/// /[entity]/[state] +/// +/// If `"entity"` and/or `"state"` are omitted, they are globbed to `"$"` (all). +#[derive(Debug, Clone, Default)] +pub struct MinionQuery { + src: String, + entity: Option, + state: Option, + scheme: String, +} + +impl MinionQuery { + pub fn new(q: &str) -> Result>, SysinspectError> { + let q = q.trim(); + if !q.starts_with(SCHEME_STATE) && !q.starts_with(SCHEME_MODEL) { + return Err(SysinspectError::ProtoError("Query has unknown scheme".to_string())); + } + + let sq: Vec<&str> = q.split("://").collect(); + if sq.len() != 2 { + return Err(SysinspectError::ProtoError("Unable to parse scheme".to_string())); + } + + let mut instance = Self { ..Default::default() }; + instance.scheme = sq[0].to_owned(); + + let sq: Vec<&str> = sq[1].split('/').filter(|s| !s.is_empty()).collect(); + match sq.len() { + 0 => { + return Err(SysinspectError::ProtoError("No model has been targeted".to_string())); + } + 1 => instance.src = sq[0].to_string(), + 2 => { + instance.src = sq[0].to_string(); + instance.entity = Some(sq[1].to_string()); + } + 3 => { + instance.src = sq[0].to_string(); + instance.entity = Some(sq[1].to_string()); + instance.state = Some(sq[2].to_string()); + } + _ => {} + } + + Ok(Arc::new(Mutex::new(instance))) + } + + pub fn target(&self) -> &str { + &self.src + } + + pub fn entity(&self) -> Option { + if let Some(entity) = &self.entity { + return Some(entity.to_owned()); + } + + None + } + + pub fn state(&self) -> Option { + if let Some(state) = &self.state { + return Some(state.to_owned()); + } + + None + } +} From ebbc871ec66c8a2361f1bfcd21f7f318754b5790 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 18 Nov 2024 18:43:30 +0100 Subject: [PATCH 137/148] Fix imports --- libsysinspect/src/proto/payload.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libsysinspect/src/proto/payload.rs b/libsysinspect/src/proto/payload.rs index 51f4b1bb..09ab9924 100644 --- a/libsysinspect/src/proto/payload.rs +++ b/libsysinspect/src/proto/payload.rs @@ -4,10 +4,9 @@ Payload types and their deserialisation. */ -use std::collections::HashMap; - use serde::{Deserialize, Serialize}; use serde_json::{from_value, Value}; +use std::collections::HashMap; /// Payload types pub enum PayloadType { From 82f0ed0c8818373cb81d15a02a3088264dc13c58 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 12:09:50 +0100 Subject: [PATCH 138/148] Change default config location --- libsysinspect/src/cfg/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libsysinspect/src/cfg/mod.rs b/libsysinspect/src/cfg/mod.rs index 285f7d66..c5de2349 100644 --- a/libsysinspect/src/cfg/mod.rs +++ b/libsysinspect/src/cfg/mod.rs @@ -10,6 +10,7 @@ use std::{env, path::PathBuf}; pub const APP_CONF: &str = "sysinspect.conf"; pub const APP_DOTCONF: &str = ".sysinspect"; +pub const APP_HOME: &str = "/etc/sysinspect"; /// Select app conf pub fn select_config(p: Option) -> Result { @@ -42,7 +43,7 @@ pub fn select_config(p: Option) -> Result { } // Global conf - let cfp = PathBuf::from(format!("/etc/{}", APP_CONF)); + let cfp = PathBuf::from(format!("{APP_HOME}/{APP_CONF}")); if cfp.exists() { return Ok(cfp); } From d9b9faf8bd1588162755cbbf440040ad439aa148 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 12:10:12 +0100 Subject: [PATCH 139/148] Clarify an error --- libsysinspect/src/inspector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsysinspect/src/inspector.rs b/libsysinspect/src/inspector.rs index a3175f41..f212843b 100644 --- a/libsysinspect/src/inspector.rs +++ b/libsysinspect/src/inspector.rs @@ -80,7 +80,7 @@ impl SysInspectRunner { } log::debug!("Done"); } - Err(err) => log::error!("Error: {}", err), + Err(err) => log::error!("Error loading mspec: {}", err), }; } } From 659b3a3af094bc778a69750719a1e2f1b65a4c41 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 17:46:29 +0100 Subject: [PATCH 140/148] Query should return the list of entities, comma-separated --- libsysinspect/src/proto/query.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libsysinspect/src/proto/query.rs b/libsysinspect/src/proto/query.rs index b21cca57..5c0af092 100644 --- a/libsysinspect/src/proto/query.rs +++ b/libsysinspect/src/proto/query.rs @@ -61,12 +61,12 @@ impl MinionQuery { &self.src } - pub fn entity(&self) -> Option { + pub fn entities(&self) -> Vec { if let Some(entity) = &self.entity { - return Some(entity.to_owned()); + return entity.split(',').map(|s| s.to_string()).collect::>(); } - None + vec![] } pub fn state(&self) -> Option { From 8b7a101fef1082b02959503f1cd8f5b6e9faed9c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 17:51:13 +0100 Subject: [PATCH 141/148] Launch sysinspect runner --- sysminion/src/minion.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index 0aa802c8..b36a9341 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -1,9 +1,11 @@ use crate::{filedata::MinionFiledata, proto, rsa::MinionRSAKeyManager}; use libsysinspect::{ cfg::{self, mmconf::MinionConfig}, + inspector::SysInspectRunner, proto::{ errcodes::ProtoErrorCode, payload::{ModStatePayload, PayloadType}, + query::MinionQuery, rqtypes::RequestType, MasterMessage, MinionMessage, ProtoConversion, }, @@ -312,12 +314,14 @@ impl SysMinion { /// Launch sysinspect async fn launch_sysinspect(self: Arc, scheme: &str, msp: &ModStatePayload) { - // TODO: Now dispatch sysinspect! - // - // 1. [x] Check if files are there, if not download them - // 2. [ ] Render the DSL according to the traits - // 3. [ ] Run the model - // 4. [ ] Collect the output and send back + // Get the query first + let mqr = match MinionQuery::new(&scheme) { + Ok(mqr) => mqr, + Err(err) => { + log::error!("Query error: {err}"); + return; + } + }; // Auto-sync all data files let cls = Arc::new(self); @@ -354,13 +358,22 @@ impl SysMinion { Err(_) => todo!(), } } - if dirty { cls.as_ptr().filedata.lock().await.init(); } + // Render DSL + + // Run the model log::debug!("Launching model for sysinspect for: {scheme}"); - // TODO: launch sysinspect here + let mqr_l = mqr.lock().unwrap(); + let mut sr = SysInspectRunner::new(); + sr.set_model_path(cls.as_ptr().cfg.models_dir().join(mqr_l.target().to_string()).to_str().unwrap_or_default()); + sr.set_state(mqr_l.state()); + sr.set_entities(mqr_l.entities()); + //sr.set_checkbook_labels(....); + sr.start(); + log::debug!("Sysinspect model cycle finished"); } async fn dispatch(self: Arc, cmd: MasterMessage) { From 9dd49ec8f940e24ea10d7f2074a3bcd937958cc2 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 18:02:54 +0100 Subject: [PATCH 142/148] Remove unnecessary double-references --- sysminion/src/minion.rs | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index b36a9341..f5a44c90 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -147,10 +147,8 @@ impl SysMinion { pub async fn do_proto(self: Arc) -> Result<(), SysinspectError> { let rstm = Arc::clone(&self.rstm); - let cls = Arc::new(self.clone()); tokio::spawn(async move { - //let mut input = BufReader::new(rstm.lock().await); loop { let mut buff = [0u8; 4]; if let Err(e) = rstm.lock().await.read_exact(&mut buff).await { @@ -193,7 +191,7 @@ impl SysMinion { log::debug!("Master sends a command"); match msg.get_retcode() { ProtoErrorCode::Success => { - let cls = cls.as_ptr().clone(); + let cls = self.as_ptr().clone(); tokio::spawn(async move { cls.dispatch(msg.to_owned()).await; }); @@ -211,7 +209,7 @@ impl SysMinion { } RequestType::Traits => { log::debug!("Master requests traits"); - if let Err(err) = cls.as_ptr().send_traits().await { + if let Err(err) = self.as_ptr().send_traits().await { log::error!("Unable to send traits: {err}"); } } @@ -225,14 +223,12 @@ impl SysMinion { std::process::exit(1); } RequestType::Ping => { - cls.request(proto::msg::get_pong()).await; + self.request(proto::msg::get_pong()).await; } _ => { log::error!("Unknown request type"); } } - - //request(wtsm_c.clone(), response).await; } }); Ok(()) @@ -315,7 +311,7 @@ impl SysMinion { /// Launch sysinspect async fn launch_sysinspect(self: Arc, scheme: &str, msp: &ModStatePayload) { // Get the query first - let mqr = match MinionQuery::new(&scheme) { + let mqr = match MinionQuery::new(scheme) { Ok(mqr) => mqr, Err(err) => { log::error!("Query error: {err}"); @@ -324,20 +320,20 @@ impl SysMinion { }; // Auto-sync all data files - let cls = Arc::new(self); let mut dirty = false; for (uri_file, fcs) in msp.files() { - let dst = cls + let dst = self + .as_ptr() .cfg .models_dir() .join(uri_file.trim_start_matches(&format!("/{}", msp.models_root())).strip_prefix("/").unwrap_or_default()); - if cls.as_ptr().filedata.lock().await.check_sha256(uri_file.to_owned(), fcs.to_owned(), true) { + if self.as_ptr().filedata.lock().await.check_sha256(uri_file.to_owned(), fcs.to_owned(), true) { continue; } log::debug!("File {uri_file} has different checksum"); - match cls.as_ptr().download_file(uri_file).await { + match self.as_ptr().download_file(uri_file).await { Ok(data) => { let dst_dir = dst.parent().unwrap(); if !dst_dir.exists() { @@ -359,7 +355,7 @@ impl SysMinion { } } if dirty { - cls.as_ptr().filedata.lock().await.init(); + self.as_ptr().filedata.lock().await.init(); } // Render DSL @@ -368,7 +364,7 @@ impl SysMinion { log::debug!("Launching model for sysinspect for: {scheme}"); let mqr_l = mqr.lock().unwrap(); let mut sr = SysInspectRunner::new(); - sr.set_model_path(cls.as_ptr().cfg.models_dir().join(mqr_l.target().to_string()).to_str().unwrap_or_default()); + sr.set_model_path(self.as_ptr().cfg.models_dir().join(mqr_l.target()).to_str().unwrap_or_default()); sr.set_state(mqr_l.state()); sr.set_entities(mqr_l.entities()); //sr.set_checkbook_labels(....); From 5faa8e40dd8254cde6199645fcb02c8a84ee1766 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 18:22:08 +0100 Subject: [PATCH 143/148] Make config:modules optional in the configuration, defaulting to /usr/share/sysinspect/modules --- libsysinspect/src/cfg/mmconf.rs | 1 + libsysinspect/src/intp/conf.rs | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libsysinspect/src/cfg/mmconf.rs b/libsysinspect/src/cfg/mmconf.rs index 6ac49a38..e508c218 100644 --- a/libsysinspect/src/cfg/mmconf.rs +++ b/libsysinspect/src/cfg/mmconf.rs @@ -11,6 +11,7 @@ pub static DEFAULT_FILESERVER_PORT: u32 = 4201; // Default directories pub static DEFAULT_SOCKET: &str = "/var/run/sysinspect-master.socket"; pub static DEFAULT_SYSINSPECT_ROOT: &str = "/etc/sysinspect"; +pub static DEFAULT_MODULES_ROOT: &str = "/usr/share/sysinspect/modules"; // All directories are relative to the sysinspect root pub static CFG_MINION_KEYS: &str = "minion-keys"; diff --git a/libsysinspect/src/intp/conf.rs b/libsysinspect/src/intp/conf.rs index c34362a5..85e04ca7 100644 --- a/libsysinspect/src/intp/conf.rs +++ b/libsysinspect/src/intp/conf.rs @@ -1,4 +1,4 @@ -use crate::{util, SysinspectError}; +use crate::{cfg::mmconf::DEFAULT_MODULES_ROOT, util, SysinspectError}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; use std::{collections::HashMap, path::PathBuf}; @@ -69,7 +69,7 @@ impl EventConfig { /// The entire config #[derive(Debug, Serialize, Deserialize, Clone, Default)] pub struct Config { - modules: PathBuf, + modules: Option, // EventId to config, added later events: Option>, @@ -87,7 +87,7 @@ impl Config { /// Get module from the namespace pub fn get_module(&self, namespace: &str) -> Result { // Fool-proof cleanup, likely a bad idea - let modpath = self.modules.join( + let modpath = &self.modules.to_owned().unwrap_or(PathBuf::from(DEFAULT_MODULES_ROOT)).join( namespace .trim_start_matches('.') .trim_end_matches('.') @@ -102,7 +102,7 @@ impl Config { return Err(SysinspectError::ModuleError(format!("Module \"{}\" was not found at {:?}", namespace, modpath))); } - Ok(modpath) + Ok(modpath.to_owned()) } /// Set events config From 2d3fc3303ae373044709cc6347efd7fb2b27f269 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 18:43:09 +0100 Subject: [PATCH 144/148] Update on config --- sysinspect.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sysinspect.conf b/sysinspect.conf index 26925596..1b056541 100644 --- a/sysinspect.conf +++ b/sysinspect.conf @@ -1,5 +1,8 @@ config: - modules: target/debug + # Path to the custom location where modules resides + # Default: /usr/share/sysinspect/modules + # + # modules: /path/to/custom/location # Configuration that is present only on master node master: From 5f165a80dc793d84b78a65650be12ebecae30a78 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 19 Nov 2024 19:10:21 +0100 Subject: [PATCH 145/148] Accept checkbook labels --- libsysinspect/src/proto/query.rs | 30 +++++++++++++++++++++++++----- sysminion/src/minion.rs | 2 +- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/libsysinspect/src/proto/query.rs b/libsysinspect/src/proto/query.rs index 5c0af092..e10dd450 100644 --- a/libsysinspect/src/proto/query.rs +++ b/libsysinspect/src/proto/query.rs @@ -11,6 +11,7 @@ pub static SCHEME_STATE: &str = "state://"; /// It has the following format: /// /// /[entity]/[state] +/// :[checkbook labels] /// /// If `"entity"` and/or `"state"` are omitted, they are globbed to `"$"` (all). #[derive(Debug, Clone, Default)] @@ -19,6 +20,7 @@ pub struct MinionQuery { entity: Option, state: Option, scheme: String, + labels: Option, } impl MinionQuery { @@ -36,7 +38,8 @@ impl MinionQuery { let mut instance = Self { ..Default::default() }; instance.scheme = sq[0].to_owned(); - let sq: Vec<&str> = sq[1].split('/').filter(|s| !s.is_empty()).collect(); + let precise = sq[1].contains('/'); + let sq: Vec<&str> = sq[1].split(if precise { '/' } else { ':' }).filter(|s| !s.is_empty()).collect(); match sq.len() { 0 => { return Err(SysinspectError::ProtoError("No model has been targeted".to_string())); @@ -44,12 +47,18 @@ impl MinionQuery { 1 => instance.src = sq[0].to_string(), 2 => { instance.src = sq[0].to_string(); - instance.entity = Some(sq[1].to_string()); + if precise { + instance.entity = Some(sq[1].to_string()); + } else { + instance.labels = Some(sq[1].to_string()); + } } 3 => { instance.src = sq[0].to_string(); - instance.entity = Some(sq[1].to_string()); - instance.state = Some(sq[2].to_string()); + if precise { + instance.entity = Some(sq[1].to_string()); + instance.state = Some(sq[2].to_string()); + } } _ => {} } @@ -57,18 +66,29 @@ impl MinionQuery { Ok(Arc::new(Mutex::new(instance))) } + /// Get target model name pub fn target(&self) -> &str { &self.src } + /// Get entities, comma-separated pub fn entities(&self) -> Vec { if let Some(entity) = &self.entity { - return entity.split(',').map(|s| s.to_string()).collect::>(); + return entity.split(',').map(|s| s.to_string()).collect(); } vec![] } + /// Get checkbook labels, comma-separated + pub fn checkbook_labels(&self) -> Vec { + if let Some(l) = &self.labels { + return l.split(',').map(|s| s.to_string()).collect(); + } + vec![] + } + + /// Get desired state of the model pub fn state(&self) -> Option { if let Some(state) = &self.state { return Some(state.to_owned()); diff --git a/sysminion/src/minion.rs b/sysminion/src/minion.rs index f5a44c90..e06b138d 100644 --- a/sysminion/src/minion.rs +++ b/sysminion/src/minion.rs @@ -367,7 +367,7 @@ impl SysMinion { sr.set_model_path(self.as_ptr().cfg.models_dir().join(mqr_l.target()).to_str().unwrap_or_default()); sr.set_state(mqr_l.state()); sr.set_entities(mqr_l.entities()); - //sr.set_checkbook_labels(....); + sr.set_checkbook_labels(mqr_l.checkbook_labels()); sr.start(); log::debug!("Sysinspect model cycle finished"); } From 435b5a45a1b949d0c52682369ce41e3b7fad5d6b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 20 Nov 2024 14:20:14 +0100 Subject: [PATCH 146/148] Updated documentation on targeting --- docs/genusage/targeting.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/genusage/targeting.rst b/docs/genusage/targeting.rst index 13493d16..c8b1d99b 100644 --- a/docs/genusage/targeting.rst +++ b/docs/genusage/targeting.rst @@ -45,18 +45,23 @@ Checkbook query is using path-like tuples to target a feature (a group of entiti in the following format: .. code-block:: text - :caption: Query synopsis + :caption: Precise model query synopsis + + "model:///[entity]/[state] [traits query]" + +.. code-block:: text + :caption: Checkbook model query synopsis - "/[entity]/[state] [traits query]" + "model:///[entity]:[checkbook labels]" Since there can be many models, it is essential to select one, therefore a model Id is always required. If ``entity`` and/or ``state`` are not specified, they are defaulted to ``$`` (all). .. code-block:: bash - :caption: Example of Model targeting + :caption: Example of Model targeting by precise query - sysinspect "router/network/online" + sysinspect "model://router/network,devices/online" In the example above, a network is verified in a router only when it supposed to be online. Under the hood, omitted parts are translated to "all" (``$``). E.g. ``router/network`` is @@ -66,6 +71,14 @@ translated as ``router/network/$``, or Model name alone e.g. ``router`` is trans Traits query is separated with a space and essentially a list of possible traits with their values and logical operations. See :ref:`query_targeting` for more details. +.. code-block:: bash + :caption: Example of Model targeting by checkbook labels + + sysinspect "model://router:network,devices" + +The example above is the same as the previous one, except it is using Checkbook. Entities +in the Checkbook are basically the top-high groups of other entities. + Using Traits ------------ From 3cbafc40efb2d347ca0084f7da01629e1ce806de Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 20 Nov 2024 14:20:32 +0100 Subject: [PATCH 147/148] Added documentation on configuration --- docs/global_config.rst | 182 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 169 insertions(+), 13 deletions(-) diff --git a/docs/global_config.rst b/docs/global_config.rst index 03f55cde..bf79dbbd 100644 --- a/docs/global_config.rst +++ b/docs/global_config.rst @@ -1,34 +1,190 @@ +.. raw:: html + + + +.. role:: u + :class: underlined + +.. role:: bi + :class: bolditalic + +.. _configuration: + Configuration ============= .. note:: - Configuration of Sysinspect + This document describes the configuration of Sysinspect system. + +Sysinspect can run in two modes: -Setup ------ +- **Distributed Mode** using network-connected Minions, allowing many "boxes" to be subscribed + to a Master command center. This is a typical use. +- **Solo Mode**, i.e. locally, only on the current local "box", affecting nothing else. This usage + is for a very small embedded systems only. -Configuration files can be in three location and are searched in the following order: +Config Location +--------------- + +Configuration files can be in three locations and are searched in the following order: 1. Current directory from which the app was launched: ``sysinspect.conf``. 2. "Dot-file" in the current user's home ``~/.sysinspect`` -3. As ``/etc/sysinspect.conf``. +3. As ``/etc/sysinspect/sysinspect.conf``. Synopsis -------- -Configuration file supports the following format: +Configuration is located under ``config`` section in a YAML file. This section +has two important sub-sections: -.. code-block:: text - :caption: Configuration Synopsis +- ``master`` for all settings of Sysinspect Master +- ``minion`` for covering settings of Sysinspect Minion - config: - : +Config Section +-------------- + +Main section of the entire configuration is ``config``. It is located at the root +of the configuration file and contains the following directives: ``modules`` -^^^^^^^^^^^ -Section ``modules`` defines the root of built-in modules: + Path to location of the modules, used in the model and states. Default + value is ``/usr/share/sysinspect/modules`` according to the LSB standard. + +``master`` + + Sysinspect Master configuration. + +``minion`` + + Sysinspect Minion configuration. + + +Master +^^^^^^ + +Sysinspect Master configuration is located under earlier mentioned ``master`` section, +and contains the following directives: + +``socket`` + + Path for a FIFO socket to communicate with the ``sysinspect`` command, + which is issuing commands over the network. + + Default value is ``/tmp/sysinspect-master.socket``. + +``bind.ip`` + + IPv4 address on which the Master is listening for all incoming and outgoing traffic + with Minion communication. + + Default value is ``0.0.0.0``. + +``bind.port`` + + Network port number on which the Master is listening using ``bind.ip`` directive. + + Sysinspect Master port is ``4200``. + + +.. important:: + + Master runs a **File Server service**. This service is :bi:`very important` for all the minions, + as they are exchanging data with the master, by downloading all the required artefacts to be + processed on their targets. + +File Server service serves static data, which is continuously checked by each minion and updated, +if that data changes. In particular, the artefacts are modules, trait configs, models, states etc. +Typically, File Server service has the root of all the data in ``/etc/sysinspect/data``. + +.. warning:: + Even though as of current version, there is no specific layout of the static data on the + File Server service to manager all the artifacts. However, this is a **subject to change**. + +Within the *"/data"* directory, *currently* one is free to organise the layout as they want. +However, it is :bi:`strongly` advised to keep all the models, states and other artefacts +separated from each other, using their own directories and namespaces. Future releases will have +configurable default namespaces for each cathegory of the artefacts. + +Below are directives for the configuration of the File Server service: + +``fileserver.bind.ip`` + + Same as ``bind.ip``, but for the internal File Server service. + +``fileserver.bind.port`` + + Network port number on which the File Server service is listening. + + File Server service port is ``4201``. + +``fileserver.models.root`` + + Relative path where are the master models kept. + +``fileserver.models`` + + List of subdirectories within ``fileserver.models.root``, exporting models. If a model is not + in the list, it will not be available for the minions. + +Example configuration for the Sysinspect Master: .. code-block:: yaml - modules: /opt/sysinspect/modules + config: + master: + socket: /tmp/sysinspect-master.socket + bind.ip: 0.0.0.0 + bind.port: 4200 + + fileserver.bind.ip: 0.0.0.0 + fileserver.bind.port: 4201 + + fileserver.models.root: /models + fileserver.models: + - my_model + - my_other_model + + +Minion +^^^^^^ + +Sysinspect Minion configuration is located under earlier mentioned ``minion`` section, +and contains the following directives: + +``root`` + + Typically, Minion if running standard, the root of all data kept by a Minion is + defaulted to ``/etc/sysinspect``, same as Master. However, in an embedded and custom + systems this might not be possible, especially if the system is usually read-only + and writable directories are limited to only a few. In this case *root* must be + set according to the system setup. + +``master.ip`` + + Corresponds to ``bind.ip`` of Master node and should be identical. + +``master.port`` + + Corresponds to ``bind.ip.port`` of Master node and should be identical. + +Example configuration for the Sysinspect Minion: + +.. code-block:: yaml + + config: + minion: + # Root directory where minion keeps all data. + # Default: /etc/sysinspect — same as for master + root: /etc/sysinspect + master.ip: 192.168.2.31 + master.port: 4200 From d7e93d639e18497ff9a9566975ffa0475cb7d430 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 20 Nov 2024 14:59:39 +0100 Subject: [PATCH 148/148] Add documentation about /etc/sysinspect layout --- docs/global_config.rst | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/global_config.rst b/docs/global_config.rst index bf79dbbd..900c54b0 100644 --- a/docs/global_config.rst +++ b/docs/global_config.rst @@ -188,3 +188,68 @@ Example configuration for the Sysinspect Minion: root: /etc/sysinspect master.ip: 192.168.2.31 master.port: 4200 + +Layout of ``/etc/sysinspect`` +----------------------------- + +Ideally, both Master and Minion have the same location of configuration and data collection, +which is defaulted to ``/etc/sysinspect``. This directory has many objects stored and has +a specific structure and purpose. For more making paths more short, this directory will be +referred as ``$SR`` *(Sysinspect Root)*. + +Common +^^^^^^ + +There are directories that are same on both Master and Minion: + +``$SR/functions`` + + Directory, containing custom trait functions. They are meant to be defined on the Master side + and then sync'ed to all the minions. + +Only on Master +^^^^^^^^^^^^^^ + +Public and private RSA keys of Master are: + +``$SR/master.rsa`` + + Master's private RSA key. + +``$SR/master.rsa.pub`` + + Master's public RSA key. + +``$SR/minion-keys`` + + Public keys from registered minions in format ``.rsa.pub``. + + Each registered minion has its own Id. Typically it is ``/etc/machine-id`` or automatically + generated one, if this file does not exist. + +``$SR/minion-registry`` + + A binary cache of minion's data, such as minion traits, data about currently connected minions etc. + This is fully purge-able directory, i.e. data can be freely deleted. However, Sysinspect Master + needs to be restarted and all minions needs to reconnect. + +Only on Minion +^^^^^^^^^^^^^^ + +Public and private RSA keys of Master are: + +``$SR/master.rsa`` + + Minion's private RSA key. + +``$SR/master.rsa.pub`` + + Minion's public RSA key. + +``$SR/traits`` + + Directory, containing custom static traits of a Minion. + +``$SR/models`` + + Directory, containing models.