Expose lazy SHA1 via watcher.unstable_lazySha1 (lazy hashing 3/3) (#1435

)

Summary:
## Stack
In this stack we're moving towards metro-file-map being able to *lazily* compute file metadata - in particular the SHA1 hash - only when required by the transformer.

More context in #1325 (comment)

## Implementing config `watcher.unstable_lazySha1`
This diff introduces a new opt-in config that
 - Disables eager computation of `sha1` for all watched files.
 - Adds support in `Transformer` to accept a callback that asynchronously returns SHA1, and optionally file content.
 - Maintains support for the old sync API, for anyone using `Transformer` directly. This will likely be dropped in a coming major.

Along with the already landed, default-on [auto-saving cache](#1434), this should provide order of magnitude[1] faster startup on large projects, with no compromise to warm build perf, and very little slowdown in cold builds in most cases[2].

[1] Metro needs to watch file subtrees, but typically only a small proportion of those files are used in a build. By hashing up front, we can spend up to several minutes hashing files that will never be used.

[2] Cold file caches with warm transform caches - typically only when using a remote cache - may be observably slower due to the need to read and hash a file that wouldn't otherwise need to be read, though this still only moves the cost from startup to build. For truly cold builds, this change adds SHA1 computation time to transform time, but requires no additional IO. SHA1 computation is typically much faster than Babel transformation, and we might consider faster algorithms in future (SHA1 is Eden-native).

Pull Request resolved: #1435

Changelog:
```
 - **[Experimental]**: Add `watcher.unstable_lazySha1` to defer SHA1 calculation until files are needed by the transformer

Differential Revision: D69373618

packages/metro-config/src/__tests__/__snapshots__/loadConfig-test.js.snap

@@ -177,6 +177,7 @@ Object {
       "debounceMs": 5000,
       "enabled": true,
     },
+    "unstable_lazySha1": false,
     "unstable_workerThreads": false,
     "watchman": Object {
       "deferStates": Array [
@@ -364,6 +365,7 @@ Object {
       "debounceMs": 5000,
       "enabled": true,
     },
+    "unstable_lazySha1": false,
     "unstable_workerThreads": false,
     "watchman": Object {
       "deferStates": Array [
@@ -551,6 +553,7 @@ Object {
       "debounceMs": 5000,
       "enabled": true,
     },
+    "unstable_lazySha1": false,
     "unstable_workerThreads": false,
     "watchman": Object {
       "deferStates": Array [
@@ -738,6 +741,7 @@ Object {
       "debounceMs": 5000,
       "enabled": true,
     },
+    "unstable_lazySha1": false,
     "unstable_workerThreads": false,
     "watchman": Object {
       "deferStates": Array [

packages/metro-config/src/configTypes.flow.js

@@ -205,6 +205,7 @@ type WatcherConfigT = {
     enabled: boolean,
     debounceMs?: number,
   }>,
+  unstable_lazySha1: boolean,
   unstable_workerThreads: boolean,
   watchman: $ReadOnly<{
     deferStates: $ReadOnlyArray<string>,

packages/metro-config/src/defaults/index.js

@@ -147,6 +147,7 @@ const getDefaultValues = (projectRoot: ?string): ConfigT => ({
       interval: 30000,
       timeout: 5000,
     },
+    unstable_lazySha1: false,
     unstable_workerThreads: false,
     unstable_autoSaveCache: {
       enabled: true,

packages/metro/src/Bundler.js

@@ -36,8 +36,16 @@ class Bundler {
       .ready()
       .then(() => {
         config.reporter.update({type: 'transformer_load_started'});
-        this._transformer = new Transformer(config, (...args) =>
-          this._depGraph.getSha1(...args),
+        this._transformer = new Transformer(
+          config,
+          config.watcher.unstable_lazySha1
+            ? // This object-form API is expected to replace passing a function
+              // once lazy SHA1 is stable. This will be a breaking change.
+              {
+                unstable_getOrComputeSha1: filePath =>
+                  this._depGraph.unstable_getOrComputeSha1(filePath),
+              }
+            : (...args) => this._depGraph.getSha1(...args),
         );
         config.reporter.update({type: 'transformer_load_done'});
       })

packages/metro/src/DeltaBundler/Transformer.js

@@ -28,15 +28,28 @@ class Transformer {
   _config: ConfigT;
   _cache: Cache<TransformResult<>>;
   _baseHash: string;
-  _getSha1: string => string;
+  _getSha1: string => string | Promise<{content?: Buffer, sha1: string}>;
   _workerFarm: WorkerFarm;
 
-  constructor(config: ConfigT, getSha1Fn: string => string) {
+  constructor(
+    config: ConfigT,
+    getSha1FnOrOpts:
+      | $ReadOnly<{
+          unstable_getOrComputeSha1: string => Promise<{
+            sha1: string,
+            content?: Buffer,
+          }>,
+        }>
+      | (string => string),
+  ) {
     this._config = config;
 
     this._config.watchFolders.forEach(verifyRootExists);
     this._cache = new Cache(config.cacheStores);
-    this._getSha1 = getSha1Fn;
+    this._getSha1 =
+      typeof getSha1FnOrOpts === 'function'
+        ? getSha1FnOrOpts
+        : getSha1FnOrOpts.unstable_getOrComputeSha1;
 
     // Remove the transformer config params that we don't want to pass to the
     // transformer. We should change the config object and move them away so we
@@ -129,11 +142,21 @@ class Transformer {
     ]);
 
     let sha1: string;
+    let content: ?Buffer;
     if (fileBuffer) {
       // Shortcut for virtual modules which provide the contents with the filename.
       sha1 = crypto.createHash('sha1').update(fileBuffer).digest('hex');
+      content = fileBuffer;
     } else {
-      sha1 = this._getSha1(filePath);
+      const result = await this._getSha1(filePath);
+      if (typeof result === 'string') {
+        sha1 = result;
+      } else {
+        sha1 = result.sha1;
+        if (result.content) {
+          content = result.content;
+        }
+      }
     }
 
     let fullKey = Buffer.concat([partialKey, Buffer.from(sha1, 'hex')]);
@@ -158,7 +181,7 @@ class Transformer {
       : await this._workerFarm.transform(
           localPath,
           transformerOptions,
-          fileBuffer,
+          content,
         );
 
     // Only re-compute the full key if the SHA-1 changed. This is because

packages/metro/src/node-haste/DependencyGraph.js

@@ -55,6 +55,14 @@ function getOrCreateMap<T>(
   return subMap;
 }
 
+const missingSha1Error = (mixedPath: string) =>
+  new Error(`Failed to get the SHA-1 for: ${mixedPath}.
+  Potential causes:
+    1) The file is not watched. Ensure it is under the configured \`projectRoot\` or \`watchFolders\`.
+    2) Check \`blockList\` in your metro.config.js and make sure it isn't excluding the file path.
+    3) The file may have been deleted since it was resolved - try refreshing your app.
+    4) Otherwise, this is a bug in Metro or the configured resolver - please report it.`);
+
 class DependencyGraph extends EventEmitter {
   _config: ConfigT;
   _haste: MetroFileMap;
@@ -258,21 +266,25 @@ class DependencyGraph extends EventEmitter {
 
   getSha1(filename: string): string {
     const sha1 = this._fileSystem.getSha1(filename);
-
     if (!sha1) {
-      throw new ReferenceError(
-        `Failed to get the SHA-1 for ${filename}.
-         Potential causes:
-           1) The file is not watched. Ensure it is under the configured \`projectRoot\` or \`watchFolders\`.
-           2) Check \`blockList\` in your metro.config.js and make sure it isn't excluding the file path.
-           3) The file may have been deleted since it was resolved - try refreshing your app.
-           4) Otherwise, this is a bug in Metro or the configured resolver - please report it.`,
-      );
+      throw missingSha1Error(filename);
     }
-
     return sha1;
   }
 
+  /**
+   * Used when watcher.unstable_lazySha1 is true
+   */
+  async unstable_getOrComputeSha1(
+    mixedPath: string,
+  ): Promise<{content?: Buffer, sha1: string}> {
+    const result = await this._fileSystem.getOrComputeSha1(mixedPath);
+    if (!result || !result.sha1) {
+      throw missingSha1Error(mixedPath);
+    }
+    return result;
+  }
+
   getWatcher(): EventEmitter {
     return this._haste;
   }

packages/metro/src/node-haste/DependencyGraph/createFileMap.js

@@ -85,7 +85,7 @@ function createFileMap(
         })),
     perfLoggerFactory: config.unstable_perfLoggerFactory,
     computeDependencies,
-    computeSha1: true,
+    computeSha1: !config.watcher.unstable_lazySha1,
     dependencyExtractor: config.resolver.dependencyExtractor,
     enableHastePackages: config?.resolver.enableGlobalPackages,
     enableSymlinks: true,