Add param to force invalidation of entire realm to from-scratch-job

Author: habdelra

packages/host/app/components/card-prerender.gts

@@ -52,9 +52,15 @@ export default class CardPrerender extends Component {
     }
   }
 
-  private async fromScratch(realmURL: URL): Promise<IndexResults> {
+  private async fromScratch(
+    realmURL: URL,
+    invalidateEntireRealm: boolean,
+  ): Promise<IndexResults> {
     try {
-      let results = await this.doFromScratch.perform(realmURL);
+      let results = await this.doFromScratch.perform(
+        realmURL,
+        invalidateEntireRealm,
+      );
       return results;
     } catch (e: any) {
       if (!didCancel(e)) {
@@ -99,21 +105,26 @@ export default class CardPrerender extends Component {
     await register(this.fromScratch.bind(this), this.incremental.bind(this));
   });
 
-  private doFromScratch = enqueueTask(async (realmURL: URL) => {
-    let { reader, indexWriter } = this.getRunnerParams(realmURL);
-    let currentRun = new CurrentRun({
-      realmURL,
-      reader,
-      indexWriter,
-      renderCard: this.renderService.renderCard,
-      render: this.renderService.render,
-    });
-    setOwner(currentRun, getOwner(this)!);
+  private doFromScratch = enqueueTask(
+    async (realmURL: URL, invalidateEntireRealm: boolean) => {
+      let { reader, indexWriter } = this.getRunnerParams(realmURL);
+      let currentRun = new CurrentRun({
+        realmURL,
+        reader,
+        indexWriter,
+        renderCard: this.renderService.renderCard,
+        render: this.renderService.render,
+      });
+      setOwner(currentRun, getOwner(this)!);
 
-    let current = await CurrentRun.fromScratch(currentRun);
-    this.renderService.indexRunDeferred?.fulfill();
-    return current;
-  });
+      let current = await CurrentRun.fromScratch(
+        currentRun,
+        invalidateEntireRealm,
+      );
+      this.renderService.indexRunDeferred?.fulfill();
+      return current;
+    },
+  );
 
   private doIncremental = enqueueTask(
     async (

packages/host/app/lib/current-run.ts

@@ -121,26 +121,50 @@ export class CurrentRun {
     this.#render = render;
   }
 
-  static async fromScratch(current: CurrentRun): Promise<IndexResults> {
+  static async fromScratch(
+    current: CurrentRun,
+    invalidateEntireRealm?: boolean,
+  ): Promise<IndexResults> {
     let start = Date.now();
     log.debug(`starting from scratch indexing`);
     console.log(
       `starting from scratch indexing for realm ${current.realmURL.href}`,
     );
 
     current.#batch = await current.#indexWriter.createBatch(current.realmURL);
-    let mtimesStart = Date.now();
-    let mtimes = await current.batch.getModifiedTimes();
-    console.log(
-      `completed getting index mtimes in ${Date.now() - mtimesStart} ms`,
-    );
-    let invalidateStart = Date.now();
-    let invalidations = (
-      await current.discoverInvalidations(current.realmURL, mtimes)
-    ).map((href) => new URL(href));
-    console.log(
-      `completed invalidations in ${Date.now() - invalidateStart} ms`,
-    );
+    let invalidations: URL[] = [];
+    if (invalidateEntireRealm) {
+      console.log(
+        `flag was set to invalidate entire realm ${current.realmURL.href}, skipping invalidation discovery`,
+      );
+      let mtimesStart = Date.now();
+      let filesystemMtimes = await current.#reader.mtimes();
+      console.log(
+        `time to get file system mtimes ${Date.now() - mtimesStart} ms`,
+      );
+      invalidations = Object.keys(filesystemMtimes)
+        .filter(
+          (url) =>
+            // Only allow json and executable files to be invalidated so that we
+            // don't end up with invalidated files that weren't meant to be indexed
+            // (images, etc)
+            url.endsWith('.json') || hasExecutableExtension(url),
+        )
+        .map((url) => new URL(url));
+    } else {
+      let mtimesStart = Date.now();
+      let mtimes = await current.batch.getModifiedTimes();
+      console.log(
+        `completed getting index mtimes in ${Date.now() - mtimesStart} ms`,
+      );
+      let invalidateStart = Date.now();
+      invalidations = (
+        await current.discoverInvalidations(current.realmURL, mtimes)
+      ).map((href) => new URL(href));
+      console.log(
+        `completed invalidations in ${Date.now() - invalidateStart} ms`,
+      );
+    }
 
     await current.whileIndexing(async () => {
       let visitStart = Date.now();
@@ -299,17 +323,9 @@ export class CurrentRun {
     if (skipList.length === 0) {
       // the whole realm needs to be visited, no need to calculate
       // invalidations--it's everything
-      console.log(
-        `entire realm ${this.realmURL.href} is invalidated--skipping invalidation calculation`,
-      );
       return invalidationList;
     }
 
-    console.log(
-      `unable to use optimized invalidation for realm ${
-        this.realmURL.href
-      }, skipped these files ${JSON.stringify(skipList)}`,
-    );
     let invalidationStart = Date.now();
     for (let invalidationURL of invalidationList) {
       await this.batch.invalidate(new URL(invalidationURL));

packages/host/app/services/local-indexer.ts

@@ -8,7 +8,10 @@ import { type TestRealmAdapter } from '@cardstack/host/tests/helpers/adapter';
 // for the test-realm-adapter
 export default class LocalIndexer extends Service {
   setup(
-    _fromScratch: (realmURL: URL) => Promise<IndexResults>,
+    _fromScratch: (
+      realmURL: URL,
+      invalidateEntireRealm: boolean,
+    ) => Promise<IndexResults>,
     _incremental: (
       url: URL,
       realmURL: URL,

packages/realm-server/server.ts

@@ -347,18 +347,21 @@ export class RealmServer {
       this.log.debug(`seed files for new realm ${url} copied to ${realmPath}`);
     }
 
-    let realm = new Realm({
-      url,
-      adapter,
-      secretSeed: this.secretSeed,
-      virtualNetwork: this.virtualNetwork,
-      dbAdapter: this.dbAdapter,
-      queue: this.queue,
-      matrix: {
-        url: this.matrixClient.matrixURL,
-        username,
+    let realm = new Realm(
+      {
+        url,
+        adapter,
+        secretSeed: this.secretSeed,
+        virtualNetwork: this.virtualNetwork,
+        dbAdapter: this.dbAdapter,
+        queue: this.queue,
+        matrix: {
+          url: this.matrixClient.matrixURL,
+          username,
+        },
       },
-    });
+      { invalidateEntireRealm: true },
+    );
     this.realms.push(realm);
     this.virtualNetwork.mount(realm.handle);
     return realm;

packages/runtime-common/realm-index-updater.ts

@@ -7,7 +7,7 @@ import {
   type Stats,
   type DBAdapter,
   type QueuePublisher,
-  type WorkerArgs,
+  type FromScratchArgs,
   type FromScratchResult,
   type IncrementalArgs,
   type IncrementalResult,
@@ -92,12 +92,13 @@ export class RealmIndexUpdater {
   // TODO consider triggering SSE events for invalidations now that we can
   // calculate fine grained invalidations for from-scratch indexing by passing
   // in an onInvalidation callback
-  async fullIndex() {
+  async fullIndex(invalidateEntireRealm?: boolean) {
     this.#indexingDeferred = new Deferred<void>();
     try {
-      let args: WorkerArgs = {
+      let args: FromScratchArgs = {
         realmURL: this.#realm.url,
         realmUsername: await this.getRealmUsername(),
+        invalidateEntireRealm: Boolean(invalidateEntireRealm),
       };
       let job = await this.#queue.publish<FromScratchResult>(
         `from-scratch-index`,

packages/runtime-common/realm.ts

@@ -163,6 +163,7 @@ export interface RealmAdapter {
 
 interface Options {
   disableModuleCaching?: true;
+  invalidateEntireRealm?: true;
 }
 
 interface UpdateItem {
@@ -252,6 +253,7 @@ export class Realm {
   #recentWrites: Map<string, number> = new Map();
   #realmSecretSeed: string;
   #disableModuleCaching = false;
+  #invalidateEntireRealm = false;
 
   #publicEndpoints: RouteTable<true> = new Map([
     [
@@ -305,6 +307,7 @@ export class Realm {
       seed: secretSeed,
     });
     this.#disableModuleCaching = Boolean(opts?.disableModuleCaching);
+    this.#invalidateEntireRealm = Boolean(opts?.invalidateEntireRealm);
 
     let fetch = fetcher(virtualNetwork.fetch, [
       async (req, next) => {
@@ -450,7 +453,7 @@ export class Realm {
   }
 
   async fullIndex() {
-    await this.realmIndexUpdater.fullIndex();
+    await this.realmIndexUpdater.fullIndex(this.#invalidateEntireRealm);
   }
 
   async flushUpdateEvents() {

packages/runtime-common/worker.ts

@@ -39,7 +39,10 @@ export interface Reader {
 }
 
 export type RunnerRegistration = (
-  fromScratch: (realmURL: URL) => Promise<IndexResults>,
+  fromScratch: (
+    realmURL: URL,
+    invalidateEntireRealm: boolean,
+  ) => Promise<IndexResults>,
   incremental: (
     url: URL,
     realmURL: URL,
@@ -72,6 +75,10 @@ export interface IncrementalResult {
   stats: Stats;
 }
 
+export interface FromScratchArgs extends WorkerArgs {
+  invalidateEntireRealm: boolean;
+}
+
 export interface FromScratchResult extends JSONTypes.Object {
   ignoreData: Record<string, string>;
   stats: Stats;
@@ -119,7 +126,11 @@ export class Worker {
   #matrixClientCache: Map<string, MatrixClient> = new Map();
   #secretSeed: string;
   #fromScratch:
-    | ((realmURL: URL, boom?: true) => Promise<IndexResults>)
+    | ((
+        realmURL: URL,
+        invalidateEntireRealm: boolean,
+        boom?: true,
+      ) => Promise<IndexResults>)
     | undefined;
   #incremental:
     | ((
@@ -239,13 +250,14 @@ export class Worker {
     return result;
   }
 
-  private fromScratch = async (args: WorkerArgs) => {
+  private fromScratch = async (args: FromScratchArgs) => {
     return await this.prepareAndRunJob<FromScratchResult>(args, async () => {
       if (!this.#fromScratch) {
         throw new Error(`Index runner has not been registered`);
       }
       let { ignoreData, stats } = await this.#fromScratch(
         new URL(args.realmURL),
+        args.invalidateEntireRealm,
       );
       return {
         ignoreData: { ...ignoreData },