diff --git a/bootstrap/bootstrap/Cargo.lock b/bootstrap/bootstrap/Cargo.lock index 8f3f5b470..d8f21893d 100644 --- a/bootstrap/bootstrap/Cargo.lock +++ b/bootstrap/bootstrap/Cargo.lock @@ -155,6 +155,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "encoding_rs" version = "0.8.32" @@ -528,6 +537,12 @@ dependencies = [ "void", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num_cpus" version = "1.15.0" @@ -576,7 +591,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -631,20 +646,26 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -877,22 +898,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "5b3e4cd94123dd520a128bcd11e34d9e9e423e7e3e50425cb1b4b1e3549d0284" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "fabfb6138d2383ea8208cf98ccf69cdfb1aff4088460681d84189aa259762f97" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.74", ] [[package]] @@ -973,6 +994,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -998,13 +1030,16 @@ dependencies = [ [[package]] name = "time" -version = "0.3.19" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa", "libc", + "num-conv", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -1012,16 +1047,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.7" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -1217,7 +1253,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-shared", ] @@ -1251,7 +1287,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/bootstrap/bootstrap/README.md b/bootstrap/bootstrap/README.md index 5374ba3a3..3ca3db384 100644 --- a/bootstrap/bootstrap/README.md +++ b/bootstrap/bootstrap/README.md @@ -17,7 +17,7 @@ will do a multiple stage bootstrap. Currently this is only two stages: ## How to works - Bootstrap is used by [0-initramfs](https://github.com/threefoldtech/0-initramfs/blob/development-zos-v3/packages/modules.sh) to basically add `internet` and `bootstrap` services to the base image -- After internet service is fully started, bootstrap will start to download flists needed to for zos node to work properly +- After internet service is fully started, bootstrap will start to download flists needed for zos node to work properly - As described above bootstrap run in two stages: - The first stage is used to update bootstrap itself, and it is done like that to avoid re-building the image if we only changed the bootstrap code. this update is basically done from `tf-autobuilder` repo in the [hub/tf-autobuilder](https://hub.grid.tf/tf-autobuilder) and download the latest bootstrap flist - For the second stage bootstrap will download the flists for that env. bootstrap cares about `runmode` argument that we pass during the start of the node. for example if we passed `runmode=dev` it will get the the tag `development` under [hub/tf-zos](https://hub.grid.tf/tf-zos) each tag is linked to a sub-directory where all flists for this env exists to be downloaded and installed on the node diff --git a/bootstrap/bootstrap/src/bootstrap.rs b/bootstrap/bootstrap/src/bootstrap.rs index 5e1c85ec8..76498c1d7 100644 --- a/bootstrap/bootstrap/src/bootstrap.rs +++ b/bootstrap/bootstrap/src/bootstrap.rs @@ -8,63 +8,16 @@ use super::workdir::WorkDir; use super::zfs::Zfs; use super::zinit; use anyhow::{Context, Result}; -use config::{RunMode, Version}; +use config::Version; use retry; const ZOS_REPO: &str = "tf-zos"; -const BIN_REPO_V2: &str = "tf-zos-bins"; -const BIN_REPO_V3: &str = "tf-zos-v3-bins"; -const FLIST_INFO_FILE: &str = "/tmp/flist.info"; -const FLIST_NAME_FILE: &str = "/tmp/flist.name"; const FLIST_TAG_FILE: &str = "/tmp/tag.info"; const BOOTSTAP_FLIST: &str = "bootstrap:latest.flist"; const WORKDIR: &str = "/tmp/bootstrap"; -fn bootstrap_zos(cfg: &config::Config) -> Result<()> { - let flist = match &cfg.runmode { - RunMode::Prod => match &cfg.version { - Version::V3 => "zos:production-3:latest.flist", - }, - RunMode::Dev => match &cfg.version { - Version::V3 => "zos:development-3:latest.flist", - }, - RunMode::Test => match &cfg.version { - Version::V3 => "zos:testing-3:latest.flist", - }, - RunMode::QA => match &cfg.version { - Version::V3 => "zos:qa-3:latest.flist", - }, - }; - - debug!("using flist: {}/{}", ZOS_REPO, flist); - let repo = hub::Repo::new(ZOS_REPO); - let flist = retry::retry(retry::delay::Exponential::from_millis(200), || { - info!("get flist info: {}", flist); - let info = match repo.get(flist) { - Ok(info) => info, - Err(err) => { - error!("failed to get info: {}", err); - bail!("failed to get info: {}", err); - } - }; - - Ok(info) - }); - - let flist = match flist { - Ok(flist) => flist, - Err(e) => bail!("failed to download flist: {:?}", e), - }; - - // write down boot info for other system components (like upgraded) - flist.write(FLIST_INFO_FILE)?; - std::fs::write(FLIST_NAME_FILE, format!("{}/{}", ZOS_REPO, flist.name))?; - - install_package(&flist) -} - /// update stage make sure we are running latest /// version of bootstrap pub fn update(_cfg: &config::Config) -> Result<()> { @@ -116,11 +69,16 @@ pub fn install(cfg: &config::Config) -> Result<()> { let repo = Repo::new(ZOS_REPO); let runmode = cfg.runmode.to_string(); - // we need to list all taglinks inside the repo + let mut listname = runmode.clone(); + match cfg.version { + Version::V3 => {} + Version::V4 => listname = format!("{}-v4", runmode), + } + // we need to list all taglinks let mut tag = None; for list in repo.list()? { - if list.kind == Kind::TagLink && list.name == runmode { + if list.kind == Kind::TagLink && list.name == listname { tag = Some(list); break; } @@ -133,12 +91,7 @@ pub fn install(cfg: &config::Config) -> Result<()> { let result = WorkDir::run(WORKDIR, || -> Result<()> { match tag { None => { - // old style bootstrap. - // we need to install binaries and zos from 2 different - // places - // we also track which binaries are installed individually - install_packages_old(cfg)?; - bootstrap_zos(cfg) + bail!("no tag found attached to this version") } Some(tag) => { // new style bootstrap @@ -160,57 +113,6 @@ pub fn install(cfg: &config::Config) -> Result<()> { result } -fn install_packages_old(cfg: &config::Config) -> Result<()> { - let name = match cfg.version { - Version::V3 => BIN_REPO_V3, - }; - - let repo = match cfg.runmode { - config::RunMode::Prod => name.into(), - config::RunMode::Dev => format!("{}.dev", name), - config::RunMode::Test => format!("{}.test", name), - config::RunMode::QA => format!("{}.qanet", name), - }; - - let client = hub::Repo::new(&repo); - let packages = retry::retry(retry::delay::Exponential::from_millis(200), || { - info!("list packages in: {}", BIN_REPO_V2); - //the full point of this match is the logging. - let packages = match client.list() { - Ok(info) => info, - Err(err) => { - error!("failed to list repo '{}': {}", BIN_REPO_V2, err); - bail!("failed to list repo '{}': {}", BIN_REPO_V2, err); - } - }; - - Ok(packages) - }); - - let packages = match packages { - Ok(packages) => packages, - Err(err) => bail!("failed to list '{}': {:?}", BIN_REPO_V2, err), - }; - - let mut map = std::collections::HashMap::new(); - for package in packages.iter() { - match install_package(package) { - Ok(_) => {} - Err(err) => warn!("failed to install package '{}': {}", package.url, err), - }; - - map.insert(format!("{}/{}", repo, package.name), package.clone()); - } - - let output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .open("/tmp/bins.info")?; - serde_json::to_writer(&output, &map)?; - - Ok(()) -} - fn install_packages(packages: &[Flist]) -> Result<()> { for package in packages { install_package(&package)?; diff --git a/bootstrap/bootstrap/src/config.rs b/bootstrap/bootstrap/src/config.rs index 1a918fc23..3e39ff23d 100644 --- a/bootstrap/bootstrap/src/config.rs +++ b/bootstrap/bootstrap/src/config.rs @@ -28,6 +28,7 @@ impl Display for RunMode { #[derive(Debug)] pub enum Version { V3, + V4, } fn runmode() -> Result { @@ -64,6 +65,7 @@ fn version() -> Result { Some(input) => match input { Some(input) => match input.as_ref() { "v3" => Version::V3, + "v4" => Version::V4, m => { bail!("unknown version: {}", m); } @@ -119,7 +121,7 @@ impl Config { } Ok(Config { - stage: stage, + stage, debug: matches.occurrences_of("debug") > 0, runmode: runmode()?, version: version()?, diff --git a/bootstrap/bootstrap/src/kparams.rs b/bootstrap/bootstrap/src/kparams.rs index d960adbd6..61e7f54be 100644 --- a/bootstrap/bootstrap/src/kparams.rs +++ b/bootstrap/bootstrap/src/kparams.rs @@ -40,10 +40,11 @@ mod tests { #[test] fn test_parse() -> Result<(), Error> { - let input: &str = "initrd=initramfs-linux.img root=UUID=10f9e7bb-ba63-4fbd-a95e-c78b5496cfbe rootflags=subvol=root rw b43.allhwsupport=1"; + let input: &str = "initrd=initramfs-linux.img version=v3 root=UUID=10f9e7bb-ba63-4fbd-a95e-c78b5496cfbe rootflags=subvol=root rw b43.allhwsupport=1"; let result = parse(input.as_bytes())?; - assert_eq!(result.len(), 5); + assert_eq!(result.len(), 6); assert_eq!(result["rw"], None); + assert_eq!(result["version"], Some(String::from("v3"))); assert_eq!(result["rootflags"], Some(String::from("subvol=root"))); Ok(()) }