Merge pull request #832 from moonbitlang/single_mbt_md_deps

feat: support deps for .mbt.md

Author: Young-Flash

.github/workflows/ci.yml

@@ -189,6 +189,10 @@ jobs:
         if: ${{ matrix.os == 'windows-latest' }}
         run: cargo run --bin moon bundle --source-dir "$env:USERPROFILE\.moon\lib\core" --all
 
+      - name: moon update
+        run: |
+          cargo run --bin moon update
+
       - name: Run tests
         run: cargo test
 
@@ -299,6 +303,10 @@ jobs:
         if: ${{ matrix.os == 'windows-latest' }}
         run: cargo run --bin moon bundle --source-dir "$env:USERPROFILE\.moon\lib\core" --all
 
+      - name: moon update
+        run: |
+          cargo run --bin moon update
+
       - name: Run tests
         run: cargo test
 

Cargo.lock

@@ -52,6 +52,7 @@ petgraph = { version = "0.6.4", features = [
 serde = { version = "1.0.163", features = ["derive"] }
 serde_json = { version = "1.0.97", features = ["preserve_order"] }
 serde_json_lenient = { version = "0.2.1", features = ["preserve_order"] }
+serde_yaml = "0.9.33"
 sysinfo = "0.29.9"
 tokio = { version = "1.34.0", features = [
     "rt-multi-thread",

Cargo.toml

@@ -24,8 +24,8 @@ use moonbuild::watcher_is_running;
 use moonbuild::{entry, MOON_PID_NAME};
 use mooncake::pkg::sync::auto_sync;
 use moonutil::cli::UniversalFlags;
-use moonutil::common::WATCH_MODE_DIR;
 use moonutil::common::{lower_surface_targets, CheckOpt};
+use moonutil::common::{parse_front_matter_config, WATCH_MODE_DIR};
 use moonutil::common::{FileLock, TargetBackend};
 use moonutil::common::{MoonbuildOpt, PrePostBuild};
 use moonutil::common::{MooncOpt, OutputFormat, RunMode, DOT_MBT_DOT_MD};
@@ -160,16 +160,27 @@ fn run_check_internal(
 }
 
 fn run_check_for_single_file(cli: &UniversalFlags, cmd: &CheckSubcommand) -> anyhow::Result<i32> {
-    let target_backend = cmd
-        .build_flags
-        .target_backend
-        .unwrap_or(TargetBackend::WasmGC);
-
     let single_file_path = &dunce::canonicalize(cmd.single_file.as_ref().unwrap()).unwrap();
     let single_file_string = single_file_path.display().to_string();
     let source_dir = single_file_path.parent().unwrap().to_path_buf();
     let raw_target_dir = source_dir.join("target");
 
+    let mbt_md_header = parse_front_matter_config(single_file_path)?;
+    let target_backend = if let Some(moonutil::common::MbtMdHeader {
+        moonbit:
+            moonutil::common::MbtMdSection {
+                backend: Some(backend),
+                ..
+            },
+    }) = &mbt_md_header
+    {
+        TargetBackend::str_to_backend(backend)?
+    } else {
+        cmd.build_flags
+            .target_backend
+            .unwrap_or(TargetBackend::WasmGC)
+    };
+
     let release_flag = !cmd.build_flags.debug;
 
     let target_dir = raw_target_dir
@@ -218,9 +229,9 @@ fn run_check_for_single_file(cli: &UniversalFlags, cmd: &CheckSubcommand) -> any
             enable_coverage: false,
             deny_warn: false,
             target_backend,
-            warn_list: None,
-            alert_list: None,
-            enable_value_tracing: false,
+            warn_list: cmd.build_flags.warn_list.clone(),
+            alert_list: cmd.build_flags.alert_list.clone(),
+            enable_value_tracing: cmd.build_flags.enable_value_tracing,
         },
         link_opt: moonutil::common::LinkCoreFlags {
             debug_flag: !release_flag,
@@ -236,9 +247,14 @@ fn run_check_for_single_file(cli: &UniversalFlags, cmd: &CheckSubcommand) -> any
         extra_build_opt: vec![],
         extra_link_opt: vec![],
         nostd: false,
-        render: true,
+        render: !cmd.build_flags.no_render,
     };
-    let module = get_module_for_single_file_test(single_file_path, &moonc_opt, &moonbuild_opt)?;
+    let module = get_module_for_single_file_test(
+        single_file_path,
+        &moonc_opt,
+        &moonbuild_opt,
+        mbt_md_header,
+    )?;
 
     if cli.dry_run {
         return dry_run::print_commands(&module, &moonc_opt, &moonbuild_opt);

crates/moon/src/cli/check.rs

@@ -32,14 +32,7 @@ pub fn install_cli(cli: UniversalFlags, _cmd: InstallSubcommand) -> anyhow::Resu
         source_dir,
         target_dir,
     } = cli.source_tgt_dir.try_into_package_dirs()?;
-    let registry_config = RegistryConfig::load();
-    mooncake::pkg::install::install(
-        &source_dir,
-        &target_dir,
-        &registry_config,
-        cli.quiet,
-        cli.verbose,
-    )
+    mooncake::pkg::install::install(&source_dir, &target_dir, cli.quiet, cli.verbose)
 }
 
 pub fn remove_cli(cli: UniversalFlags, cmd: RemoveSubcommand) -> anyhow::Result<i32> {

crates/moon/src/cli/deps.rs

@@ -232,7 +232,7 @@ pub fn generate_test_driver(
     };
 
     let module = if let Some(single_test_file) = cmd.single_test_file {
-        get_module_for_single_file_test(&single_test_file, &moonc_opt, &moonbuild_opt)?
+        get_module_for_single_file_test(&single_test_file, &moonc_opt, &moonbuild_opt, None)?
     } else {
         // Resolve dependencies, but don't download anything
         let (resolved_env, dir_sync_result) = auto_sync(

crates/moon/src/cli/generate_test_driver.rs

@@ -37,6 +37,7 @@ pub fn scan_with_x_build(
 ) -> anyhow::Result<ModuleDB> {
     let mut module = moonutil::scan::scan(
         doc_mode,
+        None,
         resolved_env,
         dir_sync_result,
         moonc_opt,
@@ -81,6 +82,7 @@ fn run_x_build(
         recreate_moon_db(&module.source_dir, &moonbuild_opt.target_dir)?;
         moonutil::scan::scan(
             false,
+            None,
             resolved_env,
             dir_sync_result,
             moonc_opt,

crates/moon/src/cli/pre_build.rs

@@ -22,14 +22,11 @@ use indexmap::IndexMap;
 use moonbuild::dry_run;
 use moonbuild::entry;
 use mooncake::pkg::sync::auto_sync;
-use moonutil::common::lower_surface_targets;
-use moonutil::common::FileLock;
-use moonutil::common::GeneratedTestDriver;
-use moonutil::common::MooncOpt;
-use moonutil::common::OutputFormat;
-use moonutil::common::RunMode;
-use moonutil::common::TargetBackend;
-use moonutil::common::{MoonbuildOpt, TestOpt};
+use mooncake::pkg::sync::auto_sync_for_single_mbt_md;
+use moonutil::common::{
+    lower_surface_targets, parse_front_matter_config, FileLock, GeneratedTestDriver, MbtMdHeader,
+    MoonbuildOpt, MooncOpt, OutputFormat, RunMode, TargetBackend, TestOpt, MOONBITLANG_CORE,
+};
 use moonutil::common::{BLACKBOX_TEST_DRIVER, DOT_MBT_DOT_MD, SINGLE_FILE_TEST_PACKAGE};
 use moonutil::cond_expr::CompileCondition;
 use moonutil::dirs::mk_arch_mode_dir;
@@ -180,16 +177,27 @@ fn run_test_internal(
 }
 
 fn run_test_in_single_file(cli: &UniversalFlags, cmd: &TestSubcommand) -> anyhow::Result<i32> {
-    let target_backend = cmd
-        .build_flags
-        .target_backend
-        .unwrap_or(TargetBackend::WasmGC);
-
     let single_file_path = &dunce::canonicalize(cmd.single_file.as_ref().unwrap()).unwrap();
     let single_file_string = single_file_path.display().to_string();
     let source_dir = single_file_path.parent().unwrap().to_path_buf();
     let raw_target_dir = source_dir.join("target");
 
+    let mbt_md_header = parse_front_matter_config(single_file_path)?;
+    let target_backend = if let Some(moonutil::common::MbtMdHeader {
+        moonbit:
+            moonutil::common::MbtMdSection {
+                backend: Some(backend),
+                ..
+            },
+    }) = &mbt_md_header
+    {
+        TargetBackend::str_to_backend(backend)?
+    } else {
+        cmd.build_flags
+            .target_backend
+            .unwrap_or(TargetBackend::WasmGC)
+    };
+
     let debug_flag = !cmd.build_flags.release;
 
     let target_dir = raw_target_dir
@@ -237,13 +245,13 @@ fn run_test_in_single_file(cli: &UniversalFlags, cmd: &TestSubcommand) -> anyhow
         build_opt: moonutil::common::BuildPackageFlags {
             debug_flag,
             strip_flag: false,
-            source_map: false,
+            source_map: debug_flag,
             enable_coverage: false,
             deny_warn: false,
             target_backend,
-            warn_list: None,
-            alert_list: None,
-            enable_value_tracing: false,
+            warn_list: cmd.build_flags.warn_list.clone(),
+            alert_list: cmd.build_flags.alert_list.clone(),
+            enable_value_tracing: cmd.build_flags.enable_value_tracing,
         },
         link_opt: moonutil::common::LinkCoreFlags {
             debug_flag,
@@ -259,9 +267,14 @@ fn run_test_in_single_file(cli: &UniversalFlags, cmd: &TestSubcommand) -> anyhow
         extra_build_opt: vec![],
         extra_link_opt: vec![],
         nostd: false,
-        render: true,
+        render: !cmd.build_flags.no_render,
     };
-    let module = get_module_for_single_file_test(single_file_path, &moonc_opt, &moonbuild_opt)?;
+    let module = get_module_for_single_file_test(
+        single_file_path,
+        &moonc_opt,
+        &moonbuild_opt,
+        mbt_md_header,
+    )?;
 
     if cli.dry_run {
         return dry_run::print_commands(&module, &moonc_opt, &moonbuild_opt);
@@ -281,6 +294,7 @@ pub fn get_module_for_single_file_test(
     single_file_path: &Path,
     moonc_opt: &MooncOpt,
     moonbuild_opt: &MoonbuildOpt,
+    front_matter_config: Option<MbtMdHeader>,
 ) -> anyhow::Result<ModuleDB> {
     let gen_single_file_test_pkg = |moonc_opt: &MooncOpt, single_file_path: &Path| -> Package {
         let path_comp = PathComponent {
@@ -335,8 +349,8 @@ pub fn get_module_for_single_file_test(
                 .join(pkg_rel_name)
                 .join(format!("{}.core", pkg_rel_name)),
             link: None,
-            warn_list: None,
-            alert_list: None,
+            warn_list: moonc_opt.build_opt.warn_list.clone(),
+            alert_list: moonc_opt.build_opt.alert_list.clone(),
             targets: None,
             pre_build: None,
             patch_file: None,
@@ -357,25 +371,51 @@ pub fn get_module_for_single_file_test(
             link_search_paths: vec![],
         }
     };
+
+    let (resolved_env, dir_sync_result, moon_mod) =
+        auto_sync_for_single_mbt_md(moonc_opt, moonbuild_opt, front_matter_config)?;
+
+    let mut module = moonutil::scan::scan(
+        false,
+        Some(moon_mod),
+        &resolved_env,
+        &dir_sync_result,
+        moonc_opt,
+        moonbuild_opt,
+    )?;
+
     let mut package = gen_single_file_test_pkg(moonc_opt, single_file_path);
     if !package.mbt_md_files.is_empty() {
         let pj_path = moonutil::doc_test::gen_md_test_patch(&package, moonc_opt)?;
         package.doc_test_patch_file = pj_path;
     }
+    let imports = module
+        .get_all_packages()
+        .iter()
+        .map(|(_, pkg)| moonutil::path::ImportComponent {
+            path: moonutil::path::ImportPath {
+                module_name: pkg.root.to_string(),
+                rel_path: pkg.rel.clone(),
+                is_3rd: true,
+            },
+            alias: None,
+            sub_package: pkg.is_sub_package,
+        })
+        // we put "moonbitlang/core/abort" in ModuleDB.packages in scan step, it's logical, so we need to filter it out
+        .filter(|import| import.path.module_name != MOONBITLANG_CORE)
+        .collect::<Vec<_>>();
+    package.imports = imports;
+
+    let packages = module.get_all_packages_mut();
+    packages.insert(package.full_name(), package.clone());
+
     let mut graph = petgraph::graph::DiGraph::new();
-    graph.add_node(package.full_name());
-
-    Ok(ModuleDB::new(
-        moonbuild_opt.source_dir.clone(),
-        "moon/test".to_string(),
-        indexmap::IndexMap::from([(package.full_name(), package)]),
-        vec![0],
-        vec![],
-        graph,
-        moonc_opt.link_opt.target_backend.to_string(),
-        "debug".to_string(),
-        None,
-    ))
+    for (_, pkg) in packages.iter() {
+        graph.add_node(pkg.full_name());
+    }
+    module.graph = graph;
+
+    Ok(module)
 }
 
 pub(crate) struct TestLikeSubcommand<'a> {
@@ -537,6 +577,7 @@ pub(crate) fn run_test_or_bench_internal(
 
     let mut module = moonutil::scan::scan(
         false,
+        None,
         &resolved_env,
         &dir_sync_result,
         &moonc_opt,