Skip to content

feat: redo how we transpile JSR packages for NPM #377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Apr 17, 2024
Merged

feat: redo how we transpile JSR packages for NPM #377

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
ce16f86
feat: redo how we transpile JSR packages for NPM
lucacasonato Apr 11, 2024
1178acd
lint
lucacasonato Apr 15, 2024
770ab47
fix
lucacasonato Apr 15, 2024
31020a3
clippy
lucacasonato Apr 15, 2024
9f95ca0
fmt
lucacasonato Apr 15, 2024
ad8289b
add unit test for relative_import_specifier
lucacasonato Apr 15, 2024
258081f
fixes
lucacasonato Apr 16, 2024
13415af
lint
lucacasonato Apr 16, 2024
620cc28
Merge remote-tracking branch 'origin/main' into redo_npm_packaging
lucacasonato Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions .env.example
View file
Open in desktop
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was duplicated with the /api/.env.example.

This file was deleted.

4 changes: 2 additions & 2 deletions api/src/analysis.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ async fn analyze_package_inner(

let npm_tarball = create_npm_tarball(NpmTarballOptions {
graph: &graph,
sources: &module_analyzer.analyzer,
analyzer: &module_analyzer.analyzer,
registry_url: registry.registry_url(),
scope: &scope,
package: &name,
Expand Down Expand Up @@ -661,7 +661,7 @@ async fn rebuild_npm_tarball_inner(

let npm_tarball = create_npm_tarball(NpmTarballOptions {
graph: &graph,
sources: &module_analyzer.analyzer,
analyzer: &module_analyzer.analyzer,
registry_url: registry.registry_url(),
scope: &scope,
package: &name,
Expand Down
246 changes: 66 additions & 180 deletions api/src/npm/emit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,215 +2,101 @@

use deno_ast::emit;
use deno_ast::fold_program;
use deno_ast::swc::ast::CallExpr;
use deno_ast::swc::ast::Callee;
use deno_ast::swc::ast::ExportAll;
use deno_ast::swc::ast::Expr;
use deno_ast::swc::ast::ExprOrSpread;
use deno_ast::swc::ast::ImportDecl;
use deno_ast::swc::ast::Lit;
use deno_ast::swc::ast::Module;
use deno_ast::swc::ast::NamedExport;
use deno_ast::swc::ast::Str;
use deno_ast::swc::common::Globals;
use deno_ast::swc::common::Mark;
use deno_ast::swc::visit::as_folder;
use deno_ast::swc::visit::noop_visit_mut_type;
use deno_ast::swc::visit::FoldWith;
use deno_ast::swc::visit::VisitMut;
use deno_ast::swc::visit::VisitMutWith;
use deno_ast::EmittedSource;
use deno_ast::ParsedSource;
use deno_ast::SourceMap;
use deno_ast::SourceMapOption;
use url::Url;
use deno_ast::TranspileOptions;
use deno_graph::FastCheckTypeModule;

use super::specifiers::rewrite_specifier;
use crate::npm::import_transform::ImportRewriteTransformer;

// todo: a lot of code is duplicated with `ParsedSource::transpile` because we
// can't fold a `ParsedSource` directly.
pub fn transpile_to_js(
source: ParsedSource,
source_url: Url,
use super::specifiers::RewriteKind;
use super::specifiers::SpecifierRewriter;

pub fn transpile_to_js<'a>(
source: &ParsedSource,
specifier_rewriter: SpecifierRewriter<'a>,
) -> Result<String, anyhow::Error> {
let transpile_options = deno_ast::TranspileOptions {
// FIXME: JSX?
..Default::default()
};
let emit_options = deno_ast::EmitOptions {
source_map: SourceMapOption::Inline,
inline_sources: true,
inline_sources: false,
keep_comments: true,
};

let file_name = source.specifier().path().split('/').last().unwrap();
let source_map =
SourceMap::single(source_url, source.text_info().text_str().to_string());
SourceMap::single(file_name, source.text_info().text_str().to_owned());

let mut folder = as_folder(NpmImportTransform);
let program = source.program_ref().clone().fold_with(&mut folder);
let mut program = source.program_ref().clone();

let mut import_rewrite_transformer = ImportRewriteTransformer {
specifier_rewriter,
kind: RewriteKind::Source,
};
program.visit_mut_with(&mut import_rewrite_transformer);

// needs to align with what's done internally in source map
assert_eq!(1, source.text_info().range().start.as_byte_pos().0);
// we need the comments to be mutable, so make it single threaded
let comments = source.comments().as_single_threaded();

let transpile_options = TranspileOptions {
use_decorators_proposal: true,
use_ts_decorators: false,

// TODO: JSX
..Default::default()
};

let globals = Globals::new();
deno_ast::swc::common::GLOBALS.set(&globals, || {
let program = deno_ast::swc::common::GLOBALS.set(&globals, || {
let top_level_mark = Mark::fresh(Mark::root());
let program = fold_program(

fold_program(
program,
&transpile_options,
&source_map,
&comments,
top_level_mark,
source.diagnostics(),
)?;
)
})?;

let emitted = emit(&program, &comments, &source_map, &emit_options)?;
let emitted = emit(&program, &comments, &source_map, &emit_options)?;

Ok(emitted.text)
})
Ok(emitted.text)
}

pub struct NpmImportTransform;

impl VisitMut for NpmImportTransform {
noop_visit_mut_type!();

fn visit_mut_module(&mut self, module: &mut Module) {
module.visit_mut_children_with(self);
}

fn visit_mut_import_decl(&mut self, node: &mut ImportDecl) {
node.visit_mut_children_with(self);

if let Some(remapped) = rewrite_specifier(&node.src.value) {
node.src = Box::new(remapped.into());
}
}

fn visit_mut_named_export(&mut self, node: &mut NamedExport) {
node.visit_mut_children_with(self);

if let Some(src) = &node.src {
if let Some(remapped) = rewrite_specifier(&src.value) {
node.src = Some(Box::new(remapped.into()));
}
}
}

fn visit_mut_export_all(&mut self, node: &mut ExportAll) {
node.visit_mut_children_with(self);

if let Some(remapped) = rewrite_specifier(&node.src.value) {
node.src = Box::new(remapped.into());
}
}

fn visit_mut_call_expr(&mut self, node: &mut CallExpr) {
node.visit_mut_children_with(self);

if let Callee::Import(_) = node.callee {
if let Some(arg) = node.args.first() {
if let Expr::Lit(Lit::Str(lit_str)) = *arg.expr.clone() {
let maybe_rewritten = rewrite_specifier(&lit_str.value);
if let Some(rewritten) = maybe_rewritten {
let replacer = Expr::Lit(Lit::Str(Str {
span: lit_str.span,
value: rewritten.into(),
raw: None,
}));
node.args[0] = ExprOrSpread {
spread: None,
expr: Box::new(replacer),
};
}
}
}
}
}
}
pub fn transpile_to_dts<'a>(
source: &ParsedSource,
fast_check_module: &FastCheckTypeModule,
specifier_rewriter: SpecifierRewriter<'a>,
) -> Result<String, anyhow::Error> {
let dts = fast_check_module.dts.as_ref().unwrap();

let emit_options = deno_ast::EmitOptions {
source_map: SourceMapOption::Inline,
inline_sources: false,
keep_comments: true,
};

let file_name = source.specifier().path().split('/').last().unwrap();
let source_map =
SourceMap::single(file_name, source.text_info().text_str().to_owned());

let mut program = dts.program.clone();

let mut import_rewrite_transformer = ImportRewriteTransformer {
specifier_rewriter,
kind: RewriteKind::Declaration,
};
program.visit_mut_with(&mut import_rewrite_transformer);

let comments = dts.comments.as_single_threaded();

let EmittedSource { text, .. } =
emit(&program, &comments, &source_map, &emit_options)?;

#[cfg(test)]
mod tests {
use deno_ast::ModuleSpecifier;
use deno_ast::ParseParams;
use deno_ast::SourceTextInfo;

use super::transpile_to_js;

fn test_transform(source: &str, expect: &str) {
let specifier = "file:///main.ts";

let parsed_source = deno_ast::parse_module(ParseParams {
specifier: ModuleSpecifier::parse(specifier).unwrap(),
media_type: deno_ast::MediaType::TypeScript,
text_info: SourceTextInfo::new(source.into()),
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})
.unwrap();

let result =
transpile_to_js(parsed_source, specifier.parse().unwrap()).unwrap();
let (result, _) = result
.rsplit_once("\n//# sourceMappingURL=data:application/json;base64,")
.unwrap();

assert_eq!(result.trim_end(), expect);
}

#[test]
fn test_transform_specifiers() {
test_transform(r#"import "./foo/bar.ts";"#, r#"import "./foo/bar.js";"#);
test_transform(r#"import "../foo.tsx";"#, r#"import "../foo.js";"#);
test_transform(r#"import "jsr:@std/path";"#, r#"import "@jsr/std__path";"#);
test_transform(r#"import "npm:@std/path";"#, r#"import "@std/path";"#);

test_transform(
"import * as foo from \"./bar.ts\";\nexport { foo };",
"import * as foo from \"./bar.js\";\nexport { foo };",
);
test_transform(
"import { asd } from \"./bar.ts\";\nexport { asd };",
"import { asd } from \"./bar.js\";\nexport { asd };",
);
test_transform(
"import { asd as foo } from \"./bar.ts\";\nexport { foo };",
"import { asd as foo } from \"./bar.js\";\nexport { foo };",
);
test_transform(
"import { asd, foo } from \"./bar.ts\";\nexport { asd, foo };",
"import { asd, foo } from \"./bar.js\";\nexport { asd, foo };",
);
test_transform(
"import asd from \"./bar.ts\";\nexport { asd };",
"import asd from \"./bar.js\";\nexport { asd };",
);
test_transform(
"import asd, { foo } from \"./bar.ts\";\nexport { asd, foo };",
"import asd, { foo } from \"./bar.js\";\nexport { asd, foo };",
);

test_transform(
"export * from \"./foo/bar.ts\";",
"export * from \"./foo/bar.js\";",
);
test_transform(
"export * as foo from \"./foo/bar.ts\";",
"export * as foo from \"./foo/bar.js\";",
);
test_transform(
"export { asd } from \"./foo/bar.ts\";",
"export { asd } from \"./foo/bar.js\";",
);
test_transform(
"export { asd as foo } from \"./foo/bar.ts\";",
"export { asd as foo } from \"./foo/bar.js\";",
);

test_transform(
"await import(\"./foo/bar.ts\");",
"await import(\"./foo/bar.js\");",
);
}
Ok(text)
}
Loading