feat: monorepo support for npx warp-drive (emberjs#9499)

Author: runspired

packages/-warp-drive/package.json

@@ -51,8 +51,10 @@
   },
   "dependencies": {
     "@embroider/macros": "^1.16.1",
+    "@manypkg/get-packages": "^2.2.1",
     "@warp-drive/build-config": "workspace:0.0.0-alpha.27",
     "semver": "^7.6.2",
+    "debug": "^4.3.5",
     "chalk": "^5.3.0",
     "comment-json": "^4.2.3"
   },

packages/-warp-drive/src/-private/shared/npm.ts

@@ -1,26 +1,36 @@
+import makeDebug from 'debug';
 import fs from 'fs';
 import { execSync } from 'node:child_process';
 import path from 'path';
 
+const debug = makeDebug('warp-drive');
+
 type NpmInfo = {
   'dist-tags': Record<string, string>;
   version: string;
   dependencies: Record<string, string>;
   devDependencies: Record<string, string>;
   peerDependencies: Record<string, string>;
   peerDependenciesMeta: Record<string, { optional?: boolean }>;
+  dist: {
+    tarball: string;
+  };
 };
 
 const InfoCache: Record<string, NpmInfo> = {};
 
 // eslint-disable-next-line @typescript-eslint/require-await
-export async function exec(cmd: string) {
-  return execSync(cmd);
+export async function exec(cmd: string, args?: Parameters<typeof execSync>[1]) {
+  debug(`exec: ${cmd}`);
+  return execSync(cmd, { ...args });
 }
 
 export async function getTags(project: string): Promise<Set<string>> {
   if (!InfoCache[project]) {
+    const start = performance.now();
     const info = await exec(`npm view ${project} --json`);
+    const end = performance.now();
+    debug(`Fetched info for ${project} in ${end - start}ms`);
     InfoCache[project] = JSON.parse(String(info)) as unknown as NpmInfo;
   }
 
@@ -30,7 +40,10 @@ export async function getTags(project: string): Promise<Set<string>> {
 
 export async function getInfo(project: string): Promise<NpmInfo> {
   if (!InfoCache[project]) {
+    const start = performance.now();
     const info = await exec(`npm view ${project} --json`);
+    const end = performance.now();
+    debug(`Fetched info for ${project} in ${end - start}ms`);
     InfoCache[project] = JSON.parse(String(info)) as unknown as NpmInfo;
   }
 

packages/-warp-drive/src/-private/shared/repo.ts

@@ -0,0 +1,10 @@
+export async function getPackageList() {
+  const { getPackages } = await import('@manypkg/get-packages');
+  const { tool, packages, rootPackage, rootDir } = await getPackages(process.cwd());
+  return {
+    pkgManager: tool.type,
+    packages,
+    rootPackage,
+    rootDir,
+  };
+}

packages/-warp-drive/src/-private/shared/utils.ts

@@ -3,6 +3,7 @@ import fs from 'fs';
 import path from 'path';
 
 import type { SEMVER_VERSION } from './channel';
+import { exec, getInfo } from './npm';
 
 /**
  * Like Pick but returns an object type instead of a union type.
@@ -272,6 +273,9 @@ export type PACKAGEJSON = {
     url: string;
     directory?: string;
   };
+  pnpm?: {
+    overrides?: Record<string, string>;
+  };
 };
 
 export function getPkgJson() {
@@ -284,14 +288,63 @@ export function writePkgJson(json: PACKAGEJSON) {
   fs.writeFileSync(path.join(process.cwd(), 'package.json'), JSON.stringify(json, null, 2) + '\n');
 }
 
-export function getLocalPkgJson(name: string) {
+const pkgJsonCache: Record<string, PACKAGEJSON> = {};
+export function getLocalPkgJson(name: string, version: string) {
+  if (pkgJsonCache[`${name}@${version}`]) {
+    return pkgJsonCache[`${name}@${version}`];
+  }
+
   const file = fs.readFileSync(path.join(process.cwd(), 'node_modules', name, 'package.json'), { encoding: 'utf-8' });
   const json = JSON.parse(file) as PACKAGEJSON;
+
+  pkgJsonCache[`${name}@${json.version}`] = json;
+  if (json.version === version) {
+    return json;
+  }
+
+  return null;
+}
+
+export async function getRemotePkgJson(name: string, version: string) {
+  if (pkgJsonCache[`${name}@${version}`]) {
+    return pkgJsonCache[`${name}@${version}`];
+  }
+
+  const dir = path.join(process.cwd(), 'tmp');
+  fs.mkdirSync(dir, { recursive: true });
+
+  // get the tarball path
+  const info = await getInfo(`${name}@${version}`);
+  const remoteTarball = info.dist.tarball;
+
+  const tarballName = `${name}-${version}`.replace('/', '_');
+
+  // download the package if needed
+  if (!fs.existsSync(path.join(dir, tarballName, 'package.json'))) {
+    await exec(`curl -L ${remoteTarball} -o ${tarballName}.tgz`, { cwd: dir });
+    await exec(`tar -xzf ${tarballName}.tgz && mv package ${tarballName}`, { cwd: dir });
+  }
+  const file = fs.readFileSync(path.join(dir, tarballName, 'package.json'), { encoding: 'utf-8' });
+  const json = JSON.parse(file) as PACKAGEJSON;
+  pkgJsonCache[`${name}@${json.version}`] = json;
   return json;
 }
 
-export function getTypePathFor(name: string) {
-  const pkg = getLocalPkgJson(name);
+export async function getPkgJsonFor(name: string, version: string) {
+  try {
+    const pkg = getLocalPkgJson(name, version);
+    if (pkg) {
+      return pkg;
+    }
+  } catch {
+    // ignore
+  }
+
+  return getRemotePkgJson(name, version);
+}
+
+export async function getTypePathFor(name: string, version: string) {
+  const pkg = await getPkgJsonFor(name, version);
   const isAlpha = pkg.files?.includes('unstable-preview-types');
   const isBeta = pkg.files?.includes('preview-types');
   const base = pkg.exports?.['.'];

packages/-warp-drive/src/-private/warp-drive/cmd.config.ts

@@ -33,7 +33,7 @@ export const RETROFIT_OPTIONS: FlagConfig = {
   help: {
     name: 'Help',
     flag: 'help',
-    flag_aliases: ['h', 'm'],
+    flag_aliases: ['h'],
     flag_mispellings: [
       'desc',
       'describe',
@@ -147,6 +147,16 @@ export const RETROFIT_OPTIONS: FlagConfig = {
       }
     },
   },
+
+  monorepo: {
+    name: 'Monorepo',
+    flag: 'monorepo',
+    flag_aliases: ['m'],
+    type: Boolean,
+    description: 'Retrofit a monorepo setup',
+    examples: [],
+    default_value: false,
+  },
 };
 
 export const COMMANDS: CommandConfig = {

packages/-warp-drive/src/-private/warp-drive/commands/retrofit.ts

@@ -32,6 +32,36 @@ export async function retrofit(flags: ParsedFlags) {
 }
 
 async function retrofitTypes(flags: Map<string, string | number | boolean | null>) {
+  if (!flags.get('monorepo')) {
+    return retrofitTypesForProject(flags);
+  }
+
+  // get the monorepo packages
+  const { getPackageList } = await import('../../shared/repo');
+  const { packages, rootDir, rootPackage, pkgManager } = await getPackageList();
+  const originalDir = process.cwd();
+
+  for (const pkg of packages) {
+    write(`Updating ${chalk.cyan(pkg.packageJson.name)}\n====================\n`);
+    process.chdir(pkg.dir);
+    await retrofitTypesForProject(flags);
+  }
+
+  write(`Updating ${chalk.cyan(rootPackage!.packageJson.name)} (<monoreporoot>)\n====================\n`);
+  process.chdir(rootDir);
+  await retrofitTypesForProject(flags, { isRoot: true, pkgManager });
+
+  const installCmd = `${pkgManager} install`;
+  await exec(installCmd);
+  write(`\t✅ Updated lockfile`);
+
+  process.chdir(originalDir);
+}
+
+async function retrofitTypesForProject(
+  flags: Map<string, string | number | boolean | null>,
+  options?: { isRoot: boolean; pkgManager: string }
+) {
   const version = flags.get('version');
   assertIsString(version);
 
@@ -232,7 +262,7 @@ async function retrofitTypes(flags: Map<string, string | number | boolean | null
 
   // add the packages to the package.json
   // and install them
-  write(chalk.grey(`\t📦  Installing ${toInstall.size} packages`));
+  write(chalk.grey(`\t📦  Updating versions for ${toInstall.size} packages`));
   if (toInstall.size > 0) {
     // add the packages to the package.json
     for (const [pkgName, config] of toInstall) {
@@ -262,6 +292,32 @@ async function retrofitTypes(flags: Map<string, string | number | boolean | null
       }
       pkg.devDependencies = sortedDeps;
     }
+    if (pkg.pnpm?.overrides) {
+      const keys = Object.keys(pkg.pnpm.overrides ?? {}).sort();
+      const sortedDeps: Record<string, string> = {};
+      for (const key of keys) {
+        sortedDeps[key] = pkg.pnpm.overrides[key];
+      }
+      pkg.pnpm.overrides = sortedDeps;
+    }
+  }
+
+  const overrideChanges = new Set<string>();
+  if (pkg.pnpm?.overrides) {
+    write(chalk.grey(`\t🔍  Checking for pnpm overrides to update`));
+    for (const pkgName of Object.keys(pkg.pnpm.overrides)) {
+      if (toInstall.has(pkgName)) {
+        const value = pkg.pnpm.overrides[pkgName];
+        const info = await getInfo(`${pkgName}@${version}`);
+        const newValue = info.version;
+
+        if (value !== newValue) {
+          overrideChanges.add(pkgName);
+          pkg.pnpm.overrides[pkgName] = newValue;
+        }
+      }
+    }
+    write(chalk.grey(`\t✅ Updated ${overrideChanges.size} pnpm overrides`));
   }
 
   const removed = new Set();
@@ -277,15 +333,31 @@ async function retrofitTypes(flags: Map<string, string | number | boolean | null
   }
   write(chalk.grey(`\t🗑  Removing ${removed.size} DefinitelyTyped packages`));
 
-  if (removed.size > 0 || toInstall.size > 0) {
+  if (removed.size > 0 || toInstall.size > 0 || overrideChanges.size > 0) {
     writePkgJson(pkg);
+    write(`\t✅ Updated package.json`);
 
     // determine which package manager to use
     // and install the packages
-    const pkgManager = getPackageManagerFromLockfile();
-    const installCmd = `${pkgManager} install`;
-    await exec(installCmd);
-    write(`\t✅ Updated package.json`);
+    if (!flags.get('monorepo')) {
+      const pkgManager = getPackageManagerFromLockfile();
+      const installCmd = `${pkgManager} install`;
+      await exec(installCmd);
+      write(`\t✅ Updated lockfile`);
+    } else {
+      write(`\t☑️ Skipped lockfile update`);
+    }
+  }
+
+  const hasAtLeastOnePackage = toInstall.size > 0 || needed.size > 0 || installed.size > 0;
+  if (!hasAtLeastOnePackage) {
+    write(`\tNo WarpDrive/EmberData packages detected`);
+    return;
+  }
+
+  if (options?.isRoot) {
+    write(chalk.grey(`\t☑️ Skipped tsconfig.json update for monorepo root`));
+    return;
   }
 
   // ensure tsconfig for each installed and needed package
@@ -296,9 +368,9 @@ async function retrofitTypes(flags: Map<string, string | number | boolean | null
     write(chalk.yellow(`\t⚠️  No tsconfig.json found in the current working directory`));
     const tsConfig = structuredClone(TS_CONFIG) as { compilerOptions: { types: string[] } };
     tsConfig.compilerOptions.types = ['ember-source/types'];
-    for (const [pkgName] of toInstall) {
+    for (const [pkgName, details] of toInstall) {
       if (Types.includes(pkgName)) {
-        const typePath = getTypePathFor(pkgName);
+        const typePath = await getTypePathFor(pkgName, details.version);
         if (!typePath) {
           throw new Error(`Could not find type path for ${pkgName}`);
         }
@@ -326,9 +398,9 @@ async function retrofitTypes(flags: Map<string, string | number | boolean | null
       tsConfig.compilerOptions.types.push('ember-source/types');
     }
 
-    for (const [pkgName] of toInstall) {
+    for (const [pkgName, details] of toInstall) {
       if (Types.includes(pkgName)) {
-        const typePath = getTypePathFor(pkgName);
+        const typePath = await getTypePathFor(pkgName, details.version);
         if (!typePath) {
           throw new Error(`Could not find type path for ${pkgName}`);
         }