diff --git a/Cargo.lock b/Cargo.lock index 0e9d1fdc481..df0f6ed8a2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9936,7 +9936,6 @@ dependencies = [ "ic-test-utilities-load-wasm", "ic-types", "ic-types-test-utils", - "ic-wasm", "ic-xrc-types", "icp-ledger", "icrc-ledger-types", diff --git a/rs/nervous_system/common/test_utils/src/wasm_helpers.rs b/rs/nervous_system/common/test_utils/src/wasm_helpers.rs index 8ceb7043f0a..124d4b3235f 100644 --- a/rs/nervous_system/common/test_utils/src/wasm_helpers.rs +++ b/rs/nervous_system/common/test_utils/src/wasm_helpers.rs @@ -1,6 +1,6 @@ -use std::io::Read; use ic_wasm; use libflate::gzip; +use std::io::Read; /// A small, valid WASM suitable for tests. pub const SMALLEST_VALID_WASM_BYTES: &[u8; 8] = &[0, 0x61, 0x73, 0x6D, 1, 0, 0, 0]; @@ -40,6 +40,8 @@ pub fn gzip_wasm(wasm: &[u8]) -> Vec { pub fn ungzip_wasm(gzipped_bytes: &[u8]) -> Vec { let mut decoder = gzip::Decoder::new(gzipped_bytes).expect("Failed to create gzip decoder."); let mut wasm_buf = Vec::new(); - decoder.read_to_end(&mut wasm_buf).expect("Failed decoding Wasm."); + decoder + .read_to_end(&mut wasm_buf) + .expect("Failed decoding Wasm."); wasm_buf } diff --git a/rs/nervous_system/integration_tests/BUILD.bazel b/rs/nervous_system/integration_tests/BUILD.bazel index 387cf6d6919..5ed2885da3c 100644 --- a/rs/nervous_system/integration_tests/BUILD.bazel +++ b/rs/nervous_system/integration_tests/BUILD.bazel @@ -29,7 +29,6 @@ BASE_DEPENDENCIES = [ "@crate_index//:assert_matches", "@crate_index//:candid", "@crate_index//:futures", - "@crate_index//:ic-wasm", "@crate_index//:itertools", "@crate_index//:lazy_static", "@crate_index//:prost", @@ -102,6 +101,7 @@ DEV_DATA = [ "//rs/sns/root:sns-root-canister", "//rs/sns/swap:sns-swap-canister", "//rs/universal_canister/impl:universal_canister.wasm.gz", + "//testnet/prebuilt-canisters:image-classification", "@cycles-ledger.wasm.gz//file", "@mainnet_ic-icrc1-archive//file", "@mainnet_ic-icrc1-index-ng//file", @@ -126,6 +126,7 @@ DEV_ENV = { "IC_ICRC1_ARCHIVE_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/archive:archive_canister)", "IC_ICRC1_INDEX_NG_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/index-ng:index_ng_canister)", "IC_ICRC1_LEDGER_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/ledger:ledger_canister)", + "IMAGE_CLASSIFICATION_CANISTER_WASM_PATH": "$(rootpath //testnet/prebuilt-canisters:image-classification)", "LEDGER_CANISTER_WASM_PATH": "$(rootpath //rs/ledger_suite/icp/ledger:ledger-canister-wasm)", "LEDGER_CANISTER_NOTIFY_METHOD_WASM_PATH": "$(rootpath //rs/ledger_suite/icp/ledger:ledger-canister-wasm-notify-method)", "LEDGER_ARCHIVE_NODE_CANISTER_WASM_PATH": "$(rootpath //rs/ledger_suite/icp/archive:ledger-archive-node-canister-wasm)", diff --git a/rs/nervous_system/integration_tests/Cargo.toml b/rs/nervous_system/integration_tests/Cargo.toml index 6815cd69c27..15a1334d584 100644 --- a/rs/nervous_system/integration_tests/Cargo.toml +++ b/rs/nervous_system/integration_tests/Cargo.toml @@ -26,7 +26,6 @@ ic-nns-governance-api = { path = "../../nns/governance/api" } ic-sns-governance = { path = "../../sns/governance" } ic-sns-root = { path = "../../sns/root" } ic-sns-swap = { path = "../../sns/swap" } -ic-wasm = { workspace = true } icp-ledger = { path = "../../ledger_suite/icp" } icrc-ledger-types = { path = "../../../packages/icrc-ledger-types" } itertools = { workspace = true } diff --git a/rs/nervous_system/integration_tests/tests/upgrade_sns_controlled_canister_with_large_wasm.rs b/rs/nervous_system/integration_tests/tests/upgrade_sns_controlled_canister_with_large_wasm.rs index a0868c38f5c..b0617beb5e0 100644 --- a/rs/nervous_system/integration_tests/tests/upgrade_sns_controlled_canister_with_large_wasm.rs +++ b/rs/nervous_system/integration_tests/tests/upgrade_sns_controlled_canister_with_large_wasm.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use candid::Principal; use canister_test::Wasm; use ic_base_types::PrincipalId; @@ -12,22 +14,16 @@ use ic_nervous_system_integration_tests::{ use ic_nervous_system_root::change_canister::ChangeCanisterRequest; use ic_nervous_system_root::change_canister::ChunkedCanisterWasm; use ic_nns_constants::ROOT_CANISTER_ID; +use ic_nns_test_utils::common::modify_wasm_bytes; use ic_sns_swap::pb::v1::Lifecycle; -use ic_test_utilities::universal_canister::UNIVERSAL_CANISTER_WASM; -use ic_wasm; use pocket_ic::nonblocking::PocketIc; use pocket_ic::PocketIcBuilder; -use ic_nervous_system_common_test_utils::wasm_helpers; const MIN_INSTALL_CHUNKED_CODE_TIME_SECONDS: u64 = 20; const MAX_INSTALL_CHUNKED_CODE_TIME_SECONDS: u64 = 5 * 60; const CHUNK_SIZE: usize = 1024 * 1024; // 1 MiB -/// This many bytes would not fit into a single cross-subnet ICP message, so including this many -/// extra bytes into a WASM module would require splitting the module into multiple chunks. -const LARGE_WASM_MIN_BYTES: usize = 2 * 1024 * 1024 + 1; - #[tokio::test] async fn test_store_same_as_target() { let store_same_as_target = true; @@ -74,29 +70,27 @@ mod interim_sns_helpers { } } -/// Produces a valid, gzipped WASM module based on `wasm`, extending it with so much junk bytes -/// that it no longer fits into ICP message limits. -/// -/// See also [`LARGE_WASM_MIN_BYTES`]. -fn oversize_wasm(wasm: Wasm) -> Wasm { - let modify_with = vec![0_u8; LARGE_WASM_MIN_BYTES]; - let mut wasm_module = ic_wasm::utils::parse_wasm(&wasm.bytes(), false).unwrap(); - // ic_wasm::metadata::add_metadata( - // &mut wasm_module, - // ic_wasm::metadata::Kind::Public, - // "aux", - // modify_with, - // ); - wasm_module.globals.add_local(); - let modified_bytes = wasm_module.emit_wasm(); - - Wasm::from_bytes(&modified_bytes[..]) +fn very_large_wasm_bytes() -> Vec { + let image_classification_canister_wasm_path = + std::env::var("IMAGE_CLASSIFICATION_CANISTER_WASM_PATH") + .expect("Please ensure that this Bazel test target correctly specifies env and data."); + + let wasm_path = std::path::PathBuf::from(image_classification_canister_wasm_path); + + std::fs::read(&wasm_path).expect("Failed to read WASM file") +} + +fn format_full_hash(hash: &[u8]) -> String { + hash.iter() + .map(|b| format!("{:02x}", b)) + .collect::>() + .join("") } /// Uploads `wasm` into the store canister, one [`CHUNK_SIZE`]-sized chunk at a time. /// /// Returns the vector of uploaded chunk hashes. -async fn upload_wasm_as_chinks( +async fn upload_wasm_as_chunks( pocket_ic: &PocketIc, store_controller_id: Principal, store_canister_id: Principal, @@ -105,33 +99,40 @@ async fn upload_wasm_as_chinks( ) -> Vec> { let sender = Some(store_controller_id); - let mut published_chunk_hashes = Vec::new(); + let mut uploaded_chunk_hashes = Vec::new(); for chunk in wasm.bytes().chunks(CHUNK_SIZE) { let uploaded_chunk_hash = pocket_ic .upload_chunk(store_canister_id, sender, chunk.to_vec()) .await .unwrap(); - - println!("uploaded_chunk_hash: {:?}", uploaded_chunk_hash); - published_chunk_hashes.push(uploaded_chunk_hash); + + uploaded_chunk_hashes.push(uploaded_chunk_hash); } // Smoke test { - let mut stored_chunk_hashes = pocket_ic + let stored_chunk_hashes = pocket_ic .stored_chunks(store_canister_id, sender) .await - .unwrap(); - stored_chunk_hashes.sort(); + .unwrap() + .into_iter() + .map(|hash| format_full_hash(&hash[..])) + .collect::>(); + + let stored_chunk_hashes = BTreeSet::from_iter(stored_chunk_hashes.iter()); - let mut published_chunk_hashes = published_chunk_hashes.clone(); - published_chunk_hashes.sort(); + let uploaded_chunk_hashes = uploaded_chunk_hashes + .iter() + .map(|hash| format_full_hash(&hash[..])) + .collect::>(); + let uploaded_chunk_hashes = BTreeSet::from_iter(uploaded_chunk_hashes.iter()); - assert_eq!(stored_chunk_hashes.len(), num_chunks_expected); + assert!(uploaded_chunk_hashes.is_subset(&stored_chunk_hashes)); + assert_eq!(uploaded_chunk_hashes.len(), num_chunks_expected); } - chunk_hashes_list + uploaded_chunk_hashes } async fn run_test(store_same_as_target: bool) { @@ -158,8 +159,7 @@ async fn run_test(store_same_as_target: bool) { }; // Install a dapp canister. - let original_wasm = wasm_helpers::ungzip_wasm(&UNIVERSAL_CANISTER_WASM.to_vec()[..]); - let original_wasm = Wasm::from_bytes(original_wasm); + let original_wasm = Wasm::from_bytes(very_large_wasm_bytes()); let original_wasm_hash = original_wasm.sha256_hash(); let app_subnet = pocket_ic.topology().await.get_app_subnets()[0]; @@ -217,34 +217,26 @@ async fn run_test(store_same_as_target: bool) { .await }; - let new_wasm = oversize_wasm(original_wasm); + let new_wasm = { + let new_wasm_bytes = modify_wasm_bytes(&original_wasm.bytes(), 42); + Wasm::from_bytes(&new_wasm_bytes[..]) + }; let new_wasm_hash = new_wasm.sha256_hash(); // Smoke test assert_ne!(new_wasm_hash, original_wasm_hash); - // We take a WASM under 1 MiB (`UNIVERSAL_CANISTER_WASM`), oversize it by adding ~2 MiB - // (`LARGE_WASM_MIN_BYTES`), then split into 1 MiB chunks. - let num_chunks_expected = 3; + // WASM with 15_843_866 bytes (`image-classification.wasm.gz`) is split into 1 MiB chunks. + let num_chunks_expected = 16; - let chunk_hashes_list = upload_wasm_as_chinks( + let chunk_hashes_list = upload_wasm_as_chunks( &pocket_ic, sns.root.canister_id.into(), store_canister_id.into(), new_wasm, num_chunks_expected, - ).await; - - println!("chunk_hashes_list = {:#?}", chunk_hashes_list); - - pocket_ic.add_cycles( - target_canister_id.into(), - 40_000_000_000_000_000 - ).await; - pocket_ic.add_cycles( - store_canister_id.into(), - 40_000_000_000_000_000 - ).await; + ) + .await; // 2. Run code under test. interim_sns_helpers::change_canister(