Handle partial failure in ISR revalidation (#410)

* fix header on HEAD requests

* handle partial failure

* Create slow-camels-swim.md

Author: conico974

.changeset/slow-camels-swim.md

@@ -0,0 +1,5 @@
+---
+"open-next": patch
+---
+
+Handle partial failure in ISR revalidation

examples/sst/stacks/OpenNextReferenceImplementation.ts

@@ -292,7 +292,12 @@ export class OpenNextCdkReferenceImplementation extends Construct {
       runtime: Runtime.NODEJS_18_X,
       timeout: Duration.seconds(30),
     });
-    consumer.addEventSource(new SqsEventSource(queue, { batchSize: 5 }));
+    consumer.addEventSource(
+      new SqsEventSource(queue, {
+        batchSize: 5,
+        reportBatchItemFailures: true,
+      }),
+    );
     return queue;
   }
 

packages/open-next/src/adapters/revalidate.ts

@@ -21,10 +21,12 @@ export interface RevalidateEvent {
   records: {
     host: string;
     url: string;
+    id: string;
   }[];
 }
 
 const defaultHandler = async (event: RevalidateEvent) => {
+  const failedRecords: RevalidateEvent["records"] = [];
   for (const record of event.records) {
     const { host, url } = record;
     debug(`Revalidating stale page`, { host, url });
@@ -37,27 +39,53 @@ const defaultHandler = async (event: RevalidateEvent) => {
     // - "previewModeId" is used to ensure the page is revalidated in a
     //   blocking way in lambda
     //   https://github.com/vercel/next.js/blob/1088b3f682cbe411be2d1edc502f8a090e36dee4/packages/next/src/server/api-utils/node.ts#L353
-    await new Promise<IncomingMessage>((resolve, reject) => {
-      const req = https.request(
-        `https://${host}${url}`,
-        {
-          method: "HEAD",
-          headers: {
-            "x-prerender-revalidate": prerenderManifest.preview.previewModeId,
-            "x-isr": "1",
+    try {
+      await new Promise<IncomingMessage>((resolve, reject) => {
+        const req = https.request(
+          `https://${host}${url}`,
+          {
+            method: "HEAD",
+            headers: {
+              "x-prerender-revalidate": prerenderManifest.preview.previewModeId,
+              "x-isr": "1",
+            },
           },
-        },
-        (res) => resolve(res),
-      );
-      req.on("error", (err) => {
-        error(`Error revalidating page`, { host, url });
-        reject(err);
+          (res) => {
+            debug("revalidating", {
+              url,
+              host,
+              headers: res.headers,
+              statusCode: res.statusCode,
+            });
+            if (
+              res.statusCode !== 200 ||
+              res.headers["x-nextjs-cache"] !== "REVALIDATED"
+            ) {
+              failedRecords.push(record);
+            }
+            resolve(res);
+          },
+        );
+        req.on("error", (err) => {
+          error(`Error revalidating page`, { host, url });
+          reject(err);
+        });
+        req.end();
       });
-      req.end();
+    } catch (err) {
+      failedRecords.push(record);
+    }
+  }
+  if (failedRecords.length > 0) {
+    error(`Failed to revalidate ${failedRecords.length} pages`, {
+      failedRecords,
     });
   }
+
   return {
     type: "revalidate",
+    // Records returned here are the ones that failed to revalidate
+    records: failedRecords,
   };
 };
 

packages/open-next/src/converters/sqs-revalidate.ts

@@ -3,20 +3,23 @@ import { Converter } from "types/open-next";
 
 import { RevalidateEvent } from "../adapters/revalidate";
 
-const converter: Converter<RevalidateEvent, { type: "revalidate" }> = {
+const converter: Converter<RevalidateEvent, RevalidateEvent> = {
   convertFrom(event: SQSEvent) {
     const records = event.Records.map((record) => {
       const { host, url } = JSON.parse(record.body);
-      return { host, url };
+      return { host, url, id: record.messageId };
     });
     return Promise.resolve({
       type: "revalidate",
       records,
     });
   },
-  convertTo() {
+  convertTo(revalidateEvent) {
     return Promise.resolve({
       type: "revalidate",
+      batchItemFailures: revalidateEvent.records.map((record) => ({
+        itemIdentifier: record.id,
+      })),
     });
   },
   name: "sqs-revalidate",

packages/open-next/src/core/routing/util.ts

@@ -56,7 +56,9 @@ export function getUrlParts(url: string, isExternal: boolean) {
 export function convertRes(res: OpenNextNodeResponse) {
   // Format Next.js response to Lambda response
   const statusCode = res.statusCode || 200;
-  const headers = parseHeaders(res.headers);
+  // When using HEAD requests, it seems that flushHeaders is not called, not sure why
+  // Probably some kind of race condition
+  const headers = parseHeaders(res.getFixedHeaders());
   const isBase64Encoded = isBinaryContentType(
     Array.isArray(headers["content-type"])
       ? headers["content-type"][0]

packages/open-next/src/http/openNextResponse.ts

@@ -145,6 +145,12 @@ export class OpenNextNodeResponse extends Transform implements ServerResponse {
     return this.headers;
   }
 
+  getFixedHeaders(): OutgoingHttpHeaders {
+    // Do we want to apply this on writeHead?
+    this.fixHeaders(this.headers);
+    return this.headers;
+  }
+
   getHeader(name: string): OutgoingHttpHeader | undefined {
     return this.headers[name.toLowerCase()];
   }
@@ -156,7 +162,6 @@ export class OpenNextNodeResponse extends Transform implements ServerResponse {
   // Only used directly in next@14+
   flushHeaders() {
     this.headersSent = true;
-    this.fixHeaders(this.headers);
     // Initial headers should be merged with the new headers
     // These initial headers are the one created either in the middleware or in next.config.js
     // We choose to override response headers with middleware headers
@@ -170,6 +175,7 @@ export class OpenNextNodeResponse extends Transform implements ServerResponse {
         ...this.initialHeaders,
       };
     }
+    this.fixHeaders(this.headers);
     if (this._cookies.length > 0) {
       // For cookies we cannot do the same as for other headers
       // We need to merge the cookies, and in this case, cookies generated by the routes or pages