feat(loader): support multi-referenced config files

Author: shigma

packages/hmr/src/index.ts

@@ -72,7 +72,8 @@ class Watcher extends Service {
   constructor(ctx: Context, public config: Watcher.Config) {
     super(ctx, 'hmr')
     this.base = resolve(ctx.baseDir, config.base || '')
-    this.initialURL = ctx.loader.file.url
+    // FIXME resolve deps based on different files
+    this.initialURL = ctx.loader.url
   }
 
   relative(filename: string) {
@@ -90,33 +91,29 @@ class Watcher extends Service {
     })
 
     // files independent from any plugins will trigger a full reload
-    const mainJob = await loader.internal!.getModuleJob('cordis/worker', this.initialURL, {})!
+    const parentURL = pathToFileURL(process.cwd()).href
+    const mainJob = await loader.internal!.getModuleJob('cordis/worker', parentURL, {})!
     this.externals = await loadDependencies(mainJob)
     const triggerLocalReload = this.ctx.debounce(() => this.triggerLocalReload(), this.config.debounce)
 
     this.watcher.on('change', async (path) => {
+      this.ctx.logger.debug('change detected:', path)
       const filename = pathToFileURL(resolve(this.base, path)).href
-      const isEntry = filename === this.initialURL
-      if (loader.file.suspend && isEntry) {
-        loader.file.suspend = false
-        return
-      }
+      if (this.externals.has(filename)) return loader.exit()
 
-      this.ctx.logger.debug('change detected:', path)
+      if (loader.internal!.loadCache.has(filename)) {
+        this.stashed.add(filename)
+        return triggerLocalReload()
+      }
 
-      if (isEntry) {
-        if (loader.internal!.loadCache.has(filename)) {
-          loader.exit()
-        } else {
-          await loader.refresh()
-        }
-      } else {
-        if (this.externals.has(filename)) {
-          loader.exit()
-        } else if (loader.internal!.loadCache.has(filename)) {
-          this.stashed.add(filename)
-          triggerLocalReload()
-        }
+      const file = this.ctx.loader.files[filename]
+      if (!file) return
+      if (file.suspend) {
+        file.suspend = false
+        return
+      }
+      for (const loader of file.groups) {
+        loader.refresh()
       }
     })
   }

packages/loader/src/entry.ts

@@ -158,9 +158,13 @@ export class Entry {
       }
       this.patch(this.fork.parent)
     } else {
-      const plugin = await this.loader.resolve(this.options.name)
-      if (!plugin) return
       const ctx = this.createContext()
+      const exports = await this.loader.import(this.options.name).catch((error: any) => {
+        ctx.emit('internal/error', new Error(`Cannot find package "${this.options.name}"`))
+        ctx.emit('internal/error', error)
+      })
+      if (!exports) return
+      const plugin = this.loader.unwrapExports(exports)
       this.patch(ctx)
       ctx[Entry.key] = this
       this.fork = ctx.plugin(plugin, this.options.config)

packages/loader/src/file.ts

@@ -6,16 +6,31 @@ import * as yaml from 'js-yaml'
 import { Entry } from './entry.ts'
 import { EntryGroup } from './group.ts'
 import { Loader } from './shared.ts'
+import { remove } from 'cosmokit'
 
 export class LoaderFile {
-  public url: string
   public suspend = false
   public mutable = false
+  public url: string
+  public groups: BaseLoader[] = []
 
   private _writeTask?: NodeJS.Timeout
 
-  constructor(public ctx: Context, public name: string, public type?: string) {
+  constructor(public loader: Loader, public name: string, public type?: string) {
     this.url = pathToFileURL(name).href
+    loader.files[name] = this
+  }
+
+  ref(group: BaseLoader) {
+    this.groups.push(group)
+    group.url = pathToFileURL(this.name).href
+  }
+
+  unref(group: BaseLoader) {
+    remove(this.groups, group)
+    if (this.groups.length) return
+    clearTimeout(this._writeTask)
+    delete this.loader.files[this.name]
   }
 
   async checkAccess() {
@@ -51,25 +66,13 @@ export class LoaderFile {
   }
 
   write(config: Entry.Options[]) {
-    this.ctx.emit('config')
+    this.loader.ctx.emit('config')
     clearTimeout(this._writeTask)
     this._writeTask = setTimeout(() => {
       this._writeTask = undefined
       this._write(config)
     }, 0)
   }
-
-  async import(name: string) {
-    if (this.ctx.loader.internal) {
-      return this.ctx.loader.internal.import(name, this.url, {})
-    } else {
-      return import(name)
-    }
-  }
-
-  dispose() {
-    clearTimeout(this._writeTask)
-  }
 }
 
 export namespace LoaderFile {
@@ -90,7 +93,9 @@ export namespace LoaderFile {
 }
 
 export class BaseLoader extends EntryGroup {
-  public file!: LoaderFile
+  static reusable = true
+
+  protected file!: LoaderFile
 
   constructor(public ctx: Context) {
     super(ctx)
@@ -107,14 +112,20 @@ export class BaseLoader extends EntryGroup {
   }
 
   stop() {
-    this.file?.dispose()
+    this.file?.unref(this)
     return super.stop()
   }
 
   write() {
     return this.file!.write(this.data)
   }
 
+  _createFile(filename: string, type: string) {
+    this.file = this.ctx.loader[filename] ??= new LoaderFile(this.ctx.loader, filename, type)
+    this.url = this.file.url
+    this.file.ref(this)
+  }
+
   async init(baseDir: string, options: Loader.Config) {
     if (options.filename) {
       const filename = resolve(baseDir, options.filename)
@@ -126,18 +137,18 @@ export class BaseLoader extends EntryGroup {
         if (!LoaderFile.supported.has(ext)) {
           throw new Error(`extension "${ext}" not supported`)
         }
-        this.file = new LoaderFile(this.ctx, filename, type)
+        this._createFile(filename, type)
       } else {
         baseDir = filename
-        await this.findConfig(baseDir, options)
+        await this._init(baseDir, options)
       }
     } else {
-      await this.findConfig(baseDir, options)
+      await this._init(baseDir, options)
     }
     this.ctx.provide('baseDir', baseDir, true)
   }
 
-  private async findConfig(baseDir: string, options: Loader.Config) {
+  private async _init(baseDir: string, options: Loader.Config) {
     const { name, initial } = options
     const dirents = await readdir(baseDir, { withFileTypes: true })
     for (const extension of LoaderFile.supported) {
@@ -148,13 +159,13 @@ export class BaseLoader extends EntryGroup {
       }
       const type = LoaderFile.writable[extension]
       const filename = resolve(baseDir, name + extension)
-      this.file = new LoaderFile(this.ctx, filename, type)
+      this._createFile(filename, type)
       return
     }
     if (initial) {
       const type = LoaderFile.writable['.yml']
       const filename = resolve(baseDir, name + '.yml')
-      this.file = new LoaderFile(this.ctx, filename, type)
+      this._createFile(filename, type)
       return this.file.write(initial as any)
     }
     throw new Error('config file not found')
@@ -175,12 +186,12 @@ export class Import extends BaseLoader {
 
   async start() {
     const { url } = this.config
-    const filename = fileURLToPath(new URL(url, this.ctx.loader.file.url))
+    const filename = fileURLToPath(new URL(url, this.ctx.scope.entry!.parent.url))
     const ext = extname(filename)
     if (!LoaderFile.supported.has(ext)) {
       throw new Error(`extension "${ext}" not supported`)
     }
-    this.file = new LoaderFile(this.ctx, filename, LoaderFile.writable[ext])
+    this._createFile(filename, LoaderFile.writable[ext])
     await super.start()
   }
 }

packages/loader/src/group.ts

@@ -1,8 +1,9 @@
 import { Context } from '@cordisjs/core'
 import { Entry } from './entry.ts'
 
-export class EntryGroup {
+export abstract class EntryGroup {
   public data: Entry.Options[] = []
+  public url!: string
 
   constructor(public ctx: Context) {
     ctx.on('dispose', () => this.stop())
@@ -76,6 +77,7 @@ export function defineGroup(config?: Entry.Options[], options: GroupOptions = {}
 
     constructor(public ctx: Context) {
       super(ctx)
+      this.url = ctx.scope.entry!.parent.url
       ctx.scope.entry!.children = this
       ctx.accept((config: Entry.Options[]) => {
         this._update(config)

packages/loader/src/shared.ts

@@ -3,7 +3,7 @@ import { Dict, isNullable, valueMap } from 'cosmokit'
 import { ModuleLoader } from './internal.ts'
 import { interpolate } from './utils.ts'
 import { Entry } from './entry.ts'
-import { BaseLoader } from './file.ts'
+import { BaseLoader, LoaderFile } from './file.ts'
 
 export * from './entry.ts'
 export * from './file.ts'
@@ -56,13 +56,12 @@ export abstract class Loader extends BaseLoader {
     env: process.env,
   }
 
+  public files: Dict<LoaderFile> = Object.create(null)
   public entries: Dict<Entry> = Object.create(null)
   public realms: Dict<Dict<symbol>> = Object.create(null)
   public delims: Dict<symbol> = Object.create(null)
   public internal?: ModuleLoader
 
-  protected tasks = new Set<Promise<any>>()
-
   constructor(public ctx: Context, public config: Loader.Config) {
     super(ctx)
     this.ctx.set('loader', this)
@@ -104,9 +103,6 @@ export abstract class Loader extends BaseLoader {
   async start() {
     await this.init(process.cwd(), this.config)
     await super.start()
-    while (this.tasks.size) {
-      await Promise.all(this.tasks)
-    }
   }
 
   interpolate(source: any) {
@@ -121,16 +117,6 @@ export abstract class Loader extends BaseLoader {
     }
   }
 
-  async resolve(name: string) {
-    const task = this.file.import(name).catch((error) => {
-      this.ctx.emit('internal/error', new Error(`Cannot find package "${name}"`))
-      this.ctx.emit('internal/error', error)
-    })
-    this.tasks.add(task)
-    task.finally(() => this.tasks.delete(task))
-    return this.unwrapExports(await task)
-  }
-
   isTruthyLike(expr: any) {
     if (isNullable(expr)) return true
     return !!this.interpolate(`\${{ ${expr} }}`)
@@ -213,6 +199,16 @@ export abstract class Loader extends BaseLoader {
     return this._locate(scope.parent.scope)
   }
 
+  async import(name: string, baseURL = this.url) {
+    if (this.internal) {
+      return this.internal.import(name, baseURL, {})
+    } else {
+      return import(name)
+    }
+  }
+
+  exit() {}
+
   unwrapExports(exports: any) {
     if (isNullable(exports)) return exports
     exports = exports.default ?? exports
@@ -222,8 +218,6 @@ export abstract class Loader extends BaseLoader {
     return exports.default ?? exports
   }
 
-  exit() {}
-
   _clearRealm(key: string, name: string) {
     const hasRef = Object.values(this.entries).some((entry) => {
       if (!entry.fork) return false

packages/loader/tests/index.spec.ts

@@ -32,7 +32,7 @@ describe('loader: basic support', () => {
         disabled: true,
       }],
     }])
-    await loader.refresh()
+    await loader.start()
 
     loader.expectEnable(foo, {})
     loader.expectEnable(bar, { a: 1 })
@@ -52,7 +52,7 @@ describe('loader: basic support', () => {
       id: '4',
       name: 'qux',
     }])
-    await loader.refresh()
+    await loader.start()
 
     loader.expectEnable(foo, {})
     loader.expectDisable(bar)

packages/loader/tests/utils.ts

@@ -24,31 +24,25 @@ class MockLoaderFile extends LoaderFile {
   write(data: Entry.Options[]) {
     this.data = data
   }
-
-  async import(name: string) {
-    return (this.ctx.loader as MockLoader).modules[name]
-  }
 }
 
 export default class MockLoader extends Loader {
-  declare file: MockLoaderFile
+  public file: MockLoaderFile
   public modules: Dict<Plugin.Object> = Object.create(null)
 
   constructor(ctx: Context) {
     super(ctx, { name: 'cordis' })
+    this.file = new MockLoaderFile(this, 'cordis.yml')
     this.mock('cordis/group', group)
   }
 
-  async refresh() {
-    await super.refresh()
-    while (this.tasks.size) {
-      await Promise.all(this.tasks)
-    }
-  }
-
   async start() {
-    this.file = new MockLoaderFile(this.ctx, 'cordis.yml')
     await this.refresh()
+    await new Promise((resolve) => setTimeout(resolve, 0))
+  }
+
+  async import(name: string) {
+    return this.modules[name]
   }
 
   mock<F extends Function>(name: string, plugin: F) {