feat: redo how we transpile JSR packages for NPM

This fixes various bugs, and introduces some features. Specifically:

- Source code is now included in the bundle to enable better "Go to source definition"
- More usecases using `/// <reference types="..." />` work correctly now, especially when d.ts file is not a sibling of the JS file.
- We strip `/// <reference types="..." />` out of JS files correctly now.
- We transpile `npm:` and `jsr:` specifiers in user provided `.d.ts` files now.

Author: lucacasonato

api/src/analysis.rs

@@ -242,7 +242,7 @@ async fn analyze_package_inner(
 
   let npm_tarball = create_npm_tarball(NpmTarballOptions {
     graph: &graph,
-    sources: &module_analyzer.analyzer,
+    analyzer: &module_analyzer.analyzer,
     registry_url: registry.registry_url(),
     scope: &scope,
     package: &name,
@@ -661,7 +661,7 @@ async fn rebuild_npm_tarball_inner(
 
   let npm_tarball = create_npm_tarball(NpmTarballOptions {
     graph: &graph,
-    sources: &module_analyzer.analyzer,
+    analyzer: &module_analyzer.analyzer,
     registry_url: registry.registry_url(),
     scope: &scope,
     package: &name,

api/src/npm/emit.rs

@@ -2,215 +2,101 @@
 
 use deno_ast::emit;
 use deno_ast::fold_program;
-use deno_ast::swc::ast::CallExpr;
-use deno_ast::swc::ast::Callee;
-use deno_ast::swc::ast::ExportAll;
-use deno_ast::swc::ast::Expr;
-use deno_ast::swc::ast::ExprOrSpread;
-use deno_ast::swc::ast::ImportDecl;
-use deno_ast::swc::ast::Lit;
-use deno_ast::swc::ast::Module;
-use deno_ast::swc::ast::NamedExport;
-use deno_ast::swc::ast::Str;
 use deno_ast::swc::common::Globals;
 use deno_ast::swc::common::Mark;
-use deno_ast::swc::visit::as_folder;
-use deno_ast::swc::visit::noop_visit_mut_type;
-use deno_ast::swc::visit::FoldWith;
-use deno_ast::swc::visit::VisitMut;
 use deno_ast::swc::visit::VisitMutWith;
+use deno_ast::EmittedSource;
 use deno_ast::ParsedSource;
 use deno_ast::SourceMap;
 use deno_ast::SourceMapOption;
-use url::Url;
+use deno_ast::TranspileOptions;
+use deno_graph::FastCheckTypeModule;
 
-use super::specifiers::rewrite_specifier;
+use crate::npm::import_transform::ImportRewriteTransformer;
 
-// todo: a lot of code is duplicated with `ParsedSource::transpile` because we
-// can't fold a `ParsedSource` directly.
-pub fn transpile_to_js(
-  source: ParsedSource,
-  source_url: Url,
+use super::specifiers::RewriteKind;
+use super::specifiers::SpecifierRewriter;
+
+pub fn transpile_to_js<'a>(
+  source: &ParsedSource,
+  specifier_rewriter: SpecifierRewriter<'a>,
 ) -> Result<String, anyhow::Error> {
-  let transpile_options = deno_ast::TranspileOptions {
-    // FIXME: JSX?
-    ..Default::default()
-  };
   let emit_options = deno_ast::EmitOptions {
     source_map: SourceMapOption::Inline,
-    inline_sources: true,
+    inline_sources: false,
     keep_comments: true,
   };
 
+  let file_name = source.specifier().path().split('/').last().unwrap();
   let source_map =
-    SourceMap::single(source_url, source.text_info().text_str().to_string());
+    SourceMap::single(file_name, source.text_info().text_str().to_owned());
 
-  let mut folder = as_folder(NpmImportTransform);
-  let program = source.program_ref().clone().fold_with(&mut folder);
+  let mut program = source.program_ref().clone();
+
+  let mut import_rewrite_transformer = ImportRewriteTransformer {
+    specifier_rewriter,
+    kind: RewriteKind::Source,
+  };
+  program.visit_mut_with(&mut import_rewrite_transformer);
 
-  // needs to align with what's done internally in source map
-  assert_eq!(1, source.text_info().range().start.as_byte_pos().0);
-  // we need the comments to be mutable, so make it single threaded
   let comments = source.comments().as_single_threaded();
+
+  let transpile_options = TranspileOptions {
+    use_decorators_proposal: true,
+    use_ts_decorators: false,
+
+    // TODO: JSX
+    ..Default::default()
+  };
+
   let globals = Globals::new();
-  deno_ast::swc::common::GLOBALS.set(&globals, || {
+  let program = deno_ast::swc::common::GLOBALS.set(&globals, || {
     let top_level_mark = Mark::fresh(Mark::root());
-    let program = fold_program(
+
+    fold_program(
       program,
       &transpile_options,
       &source_map,
       &comments,
       top_level_mark,
       source.diagnostics(),
-    )?;
+    )
+  })?;
 
-    let emitted = emit(&program, &comments, &source_map, &emit_options)?;
+  let emitted = emit(&program, &comments, &source_map, &emit_options)?;
 
-    Ok(emitted.text)
-  })
+  Ok(emitted.text)
 }
 
-pub struct NpmImportTransform;
-
-impl VisitMut for NpmImportTransform {
-  noop_visit_mut_type!();
-
-  fn visit_mut_module(&mut self, module: &mut Module) {
-    module.visit_mut_children_with(self);
-  }
-
-  fn visit_mut_import_decl(&mut self, node: &mut ImportDecl) {
-    node.visit_mut_children_with(self);
-
-    if let Some(remapped) = rewrite_specifier(&node.src.value) {
-      node.src = Box::new(remapped.into());
-    }
-  }
-
-  fn visit_mut_named_export(&mut self, node: &mut NamedExport) {
-    node.visit_mut_children_with(self);
-
-    if let Some(src) = &node.src {
-      if let Some(remapped) = rewrite_specifier(&src.value) {
-        node.src = Some(Box::new(remapped.into()));
-      }
-    }
-  }
-
-  fn visit_mut_export_all(&mut self, node: &mut ExportAll) {
-    node.visit_mut_children_with(self);
-
-    if let Some(remapped) = rewrite_specifier(&node.src.value) {
-      node.src = Box::new(remapped.into());
-    }
-  }
-
-  fn visit_mut_call_expr(&mut self, node: &mut CallExpr) {
-    node.visit_mut_children_with(self);
-
-    if let Callee::Import(_) = node.callee {
-      if let Some(arg) = node.args.first() {
-        if let Expr::Lit(Lit::Str(lit_str)) = *arg.expr.clone() {
-          let maybe_rewritten = rewrite_specifier(&lit_str.value);
-          if let Some(rewritten) = maybe_rewritten {
-            let replacer = Expr::Lit(Lit::Str(Str {
-              span: lit_str.span,
-              value: rewritten.into(),
-              raw: None,
-            }));
-            node.args[0] = ExprOrSpread {
-              spread: None,
-              expr: Box::new(replacer),
-            };
-          }
-        }
-      }
-    }
-  }
-}
+pub fn transpile_to_dts<'a>(
+  source: &ParsedSource,
+  fast_check_module: &FastCheckTypeModule,
+  specifier_rewriter: SpecifierRewriter<'a>,
+) -> Result<String, anyhow::Error> {
+  let dts = fast_check_module.dts.as_ref().unwrap();
+
+  let emit_options = deno_ast::EmitOptions {
+    source_map: SourceMapOption::Inline,
+    inline_sources: false,
+    keep_comments: true,
+  };
+
+  let file_name = source.specifier().path().split('/').last().unwrap();
+  let source_map =
+    SourceMap::single(file_name, source.text_info().text_str().to_owned());
+
+  let mut program = dts.program.clone();
+
+  let mut import_rewrite_transformer = ImportRewriteTransformer {
+    specifier_rewriter,
+    kind: RewriteKind::Declaration,
+  };
+  program.visit_mut_with(&mut import_rewrite_transformer);
+
+  let comments = dts.comments.as_single_threaded();
+
+  let EmittedSource { text, .. } =
+    emit(&program, &comments, &source_map, &emit_options)?;
 
-#[cfg(test)]
-mod tests {
-  use deno_ast::ModuleSpecifier;
-  use deno_ast::ParseParams;
-  use deno_ast::SourceTextInfo;
-
-  use super::transpile_to_js;
-
-  fn test_transform(source: &str, expect: &str) {
-    let specifier = "file:///main.ts";
-
-    let parsed_source = deno_ast::parse_module(ParseParams {
-      specifier: ModuleSpecifier::parse(specifier).unwrap(),
-      media_type: deno_ast::MediaType::TypeScript,
-      text_info: SourceTextInfo::new(source.into()),
-      capture_tokens: false,
-      scope_analysis: false,
-      maybe_syntax: None,
-    })
-    .unwrap();
-
-    let result =
-      transpile_to_js(parsed_source, specifier.parse().unwrap()).unwrap();
-    let (result, _) = result
-      .rsplit_once("\n//# sourceMappingURL=data:application/json;base64,")
-      .unwrap();
-
-    assert_eq!(result.trim_end(), expect);
-  }
-
-  #[test]
-  fn test_transform_specifiers() {
-    test_transform(r#"import "./foo/bar.ts";"#, r#"import "./foo/bar.js";"#);
-    test_transform(r#"import "../foo.tsx";"#, r#"import "../foo.js";"#);
-    test_transform(r#"import "jsr:@std/path";"#, r#"import "@jsr/std__path";"#);
-    test_transform(r#"import "npm:@std/path";"#, r#"import "@std/path";"#);
-
-    test_transform(
-      "import * as foo from \"./bar.ts\";\nexport { foo };",
-      "import * as foo from \"./bar.js\";\nexport { foo };",
-    );
-    test_transform(
-      "import { asd } from \"./bar.ts\";\nexport { asd };",
-      "import { asd } from \"./bar.js\";\nexport { asd };",
-    );
-    test_transform(
-      "import { asd as foo } from \"./bar.ts\";\nexport { foo };",
-      "import { asd as foo } from \"./bar.js\";\nexport { foo };",
-    );
-    test_transform(
-      "import { asd, foo } from \"./bar.ts\";\nexport { asd, foo };",
-      "import { asd, foo } from \"./bar.js\";\nexport { asd, foo };",
-    );
-    test_transform(
-      "import asd from \"./bar.ts\";\nexport { asd };",
-      "import asd from \"./bar.js\";\nexport { asd };",
-    );
-    test_transform(
-      "import asd, { foo } from \"./bar.ts\";\nexport { asd, foo };",
-      "import asd, { foo } from \"./bar.js\";\nexport { asd, foo };",
-    );
-
-    test_transform(
-      "export * from \"./foo/bar.ts\";",
-      "export * from \"./foo/bar.js\";",
-    );
-    test_transform(
-      "export * as foo from \"./foo/bar.ts\";",
-      "export * as foo from \"./foo/bar.js\";",
-    );
-    test_transform(
-      "export { asd } from \"./foo/bar.ts\";",
-      "export { asd } from \"./foo/bar.js\";",
-    );
-    test_transform(
-      "export { asd as foo } from \"./foo/bar.ts\";",
-      "export { asd as foo } from \"./foo/bar.js\";",
-    );
-
-    test_transform(
-      "await import(\"./foo/bar.ts\");",
-      "await import(\"./foo/bar.js\");",
-    );
-  }
+  Ok(text)
 }

api/src/npm/import_transform.rs

@@ -0,0 +1,97 @@
+use deno_ast::swc::ast::CallExpr;
+use deno_ast::swc::ast::Callee;
+use deno_ast::swc::ast::ExportAll;
+use deno_ast::swc::ast::Expr;
+use deno_ast::swc::ast::ExprOrSpread;
+use deno_ast::swc::ast::ImportDecl;
+use deno_ast::swc::ast::Lit;
+use deno_ast::swc::ast::Module;
+use deno_ast::swc::ast::NamedExport;
+use deno_ast::swc::ast::Str;
+use deno_ast::swc::ast::TsImportType;
+use deno_ast::swc::visit::VisitMut;
+use deno_ast::swc::visit::VisitMutWith;
+
+use super::specifiers::RewriteKind;
+use super::specifiers::SpecifierRewriter;
+
+pub struct ImportRewriteTransformer<'a> {
+  pub specifier_rewriter: SpecifierRewriter<'a>,
+  pub kind: RewriteKind,
+}
+
+impl<'a> VisitMut for ImportRewriteTransformer<'a> {
+  fn visit_mut_module(&mut self, module: &mut Module) {
+    module.visit_mut_children_with(self);
+  }
+
+  fn visit_mut_import_decl(&mut self, node: &mut ImportDecl) {
+    node.visit_mut_children_with(self);
+
+    if let Some(remapped) = self
+      .specifier_rewriter
+      .rewrite(&node.src.value.as_str(), self.kind)
+    {
+      node.src = Box::new(remapped.into());
+    }
+  }
+
+  fn visit_mut_named_export(&mut self, node: &mut NamedExport) {
+    node.visit_mut_children_with(self);
+
+    if let Some(src) = &node.src {
+      if let Some(remapped) = self
+        .specifier_rewriter
+        .rewrite(&src.value.as_str(), self.kind)
+      {
+        node.src = Some(Box::new(remapped.into()));
+      }
+    }
+  }
+
+  fn visit_mut_export_all(&mut self, node: &mut ExportAll) {
+    node.visit_mut_children_with(self);
+
+    if let Some(remapped) = self
+      .specifier_rewriter
+      .rewrite(&node.src.value.as_str(), self.kind)
+    {
+      node.src = Box::new(remapped.into());
+    }
+  }
+
+  fn visit_mut_ts_import_type(&mut self, n: &mut TsImportType) {
+    n.visit_mut_children_with(self);
+
+    if let Some(remapped) = self
+      .specifier_rewriter
+      .rewrite(&n.arg.value.as_str(), RewriteKind::Declaration)
+    {
+      n.arg = remapped.into();
+    }
+  }
+
+  fn visit_mut_call_expr(&mut self, node: &mut CallExpr) {
+    node.visit_mut_children_with(self);
+
+    if let Callee::Import(_) = node.callee {
+      if let Some(arg) = node.args.first() {
+        if let Expr::Lit(Lit::Str(lit_str)) = *arg.expr.clone() {
+          let maybe_rewritten =
+            self.specifier_rewriter.rewrite(&lit_str.value, self.kind);
+          if let Some(rewritten) = maybe_rewritten {
+            let replacer = Expr::Lit(Lit::Str(Str {
+              span: lit_str.span,
+              value: rewritten.into(),
+              raw: None,
+            }));
+            node.args[0] = ExprOrSpread {
+              spread: None,
+              expr: Box::new(replacer),
+            };
+          }
+        }
+      }
+    }
+  }
+}