fix(uploadBigFile): workaround timeouts when closing big file

Author: ido-pluto

examples/simple-form/src/middleware.ts

@@ -5,7 +5,7 @@ export const onRequest = sequence(astroForms({
     forms: {
         bigFilesUpload: {
             bigFileServerOptions: {
-                maxUploadSize: 1024 * 1024 * 1024, // 1GB
+                maxUploadSize: 1024 * 1024 * 1024 * 2.5, // 2.5GB
                 maxDirectorySize: 1024 * 1024 * 1024 * 10, // 10GB
             }
         }

packages/forms/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "scripts": {
     "watch": "onchange 'src/**/*' -- npm run build",
-    "build": "rm -r dist/*; tsc; mkdir dist/components; cp -r src/components/* dist/components/",
+    "build": "rm -r dist/*; tsc; mkdir dist/components; cp -r src/components/* dist/components/; find  dist/components/ -name '*.ts' -delete ",
     "prepack": "npm run build"
   },
   "keywords": [

packages/forms/src/components/form/UploadBigFile/uploadBigFileClient.ts

@@ -1,5 +1,7 @@
 import { v4 as uuid } from 'uuid';
 
+const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
+
 type ProgressCallback = (progress: number, total: number) => void;
 
 export type BigFileUploadOptions = {
@@ -8,6 +10,7 @@ export type BigFileUploadOptions = {
     chunkSize: number;
     parallelChunks: number;
     parallelUploads: number;
+    waitFinishDelay?: number;
 };
 
 const UPLOAD_BIG_FILE_OPTIONS: BigFileUploadOptions = {
@@ -16,6 +19,7 @@ const UPLOAD_BIG_FILE_OPTIONS: BigFileUploadOptions = {
     chunkSize: 1024 * 1024 * 5,
     parallelChunks: 3,
     parallelUploads: 3,
+    waitFinishDelay: 1000,
 };
 
 const clientWFS = (window as any).clientWFS;
@@ -54,6 +58,52 @@ async function uploadChunkWithXHR(file: Blob, info: Record<string, any>, progres
     });
 }
 
+async function finishUpload(uploadId: string, options: BigFileUploadOptions) {
+    let maxError = options.retryChunks;
+    while (true) {
+        try {
+            const response = await new Promise<any>((resolve, reject) => {
+                const xhr = new XMLHttpRequest();
+                const formData = new FormData();
+                xhr.responseType = "text";
+
+                formData.append('wait', uploadId);
+                formData.append("astroBigFileUpload", "true");
+
+                if (clientWFS.csrf) {
+                    formData.append(clientWFS.csrf.filed, clientWFS.csrf.token);
+                }
+
+                xhr.onload = () => {
+                    if (xhr.status >= 200 && xhr.status < 300) {
+                        resolve(JSON.parse(xhr.responseText));
+                    } else {
+                        reject({ ok: false, error: xhr.responseText });
+                    }
+                };
+                xhr.onerror = () => {
+                    reject({ ok: false, error: xhr.responseText });
+                };
+
+                xhr.open('POST', location.href, true);
+                xhr.send(formData);
+            });
+
+            if (!response.wait) {
+                break;
+            }
+
+            await sleep(options.waitFinishDelay);
+        } catch (error) {
+            if(maxError === 0){
+                throw error;
+            }
+            maxError--;
+            await sleep(options.retryChunks);
+        }
+    }
+}
+
 async function uploadBigFile(fileId: string, file: File, progressCallback: ProgressCallback, options: BigFileUploadOptions) {
     const totalSize = file.size;
     const totalChunks = Math.ceil(totalSize / options.chunkSize);
@@ -119,6 +169,7 @@ async function uploadBigFile(fileId: string, file: File, progressCallback: Progr
     }
 
     await Promise.all(activeChunks);
+    await finishUpload(fileId, options);
 }
 
 export async function uploadAllFiles(els: NodeListOf<HTMLInputElement>, options: BigFileUploadOptions = { ...UPLOAD_BIG_FILE_OPTIONS, ...clientWFS.bigFileUploadOptions }) {
@@ -231,7 +282,7 @@ async function retry(fn: () => Promise<void>, options: { retries: number, delay:
             if (attempts >= options.retries) {
                 throw error;
             }
-            await new Promise(res => setTimeout(res, options.delay));
+            await sleep(options.delay);
         }
     }
 }

packages/forms/src/components/form/UploadBigFile/uploadBigFileServer.ts

@@ -43,6 +43,8 @@ export const DEFAULT_BIG_FILE_UPLOAD_OPTIONS_SERVER: LoadUploadFilesOptions = {
     tempDirectory: path.join(os.tmpdir(), "astro_forms_big_files_uploads"),
 };
 
+const ACTIVE_FINISHED_UPLOADS = new Set<string>();
+
 async function loadUploadFiles(astro: AstroGlobal, options: Partial<LoadUploadFilesOptions> = {}) {
     const { allowUpload, onFinished, maxUploadTime, maxUploadSize, maxDirectorySize, tempDirectory } = { ...DEFAULT_BIG_FILE_UPLOAD_OPTIONS_SERVER, ...options };
     if (astro.request.method !== "POST" || !await validateFrom(astro)) {
@@ -53,6 +55,13 @@ async function loadUploadFiles(astro: AstroGlobal, options: Partial<LoadUploadFi
         return false;
     }
 
+    const hasWait = await getFormValue(astro.request, "wait");
+    if (hasWait) {
+        const thisWait = String(hasWait);
+        return Response.json({ ok: true, wait: ACTIVE_FINISHED_UPLOADS.has(thisWait) });
+    }
+
+
     await fsExtra.ensureDir(tempDirectory);
     await deleteOldUploads(tempDirectory, maxUploadTime);
     const uploadInfo = await getFormValue(astro.request, "info");
@@ -127,28 +136,36 @@ async function loadUploadFiles(astro: AstroGlobal, options: Partial<LoadUploadFi
         return await sendError(`Missing chunks ${missingChunks}, upload failed`, false, { missingChunks });
     }
 
-    const outputStream = oldFs.createWriteStream(uploadFilePath, { flags: 'a' });
-    for (let i = 1; i <= total; i++) {
-        const fileFullPath = path.join(uploadDir, `${i}-${total}`);
-        const inputStream = oldFs.createReadStream(fileFullPath);
-        await new Promise((resolve, reject) => {
-            inputStream.on("data", (chunk) => {
-                outputStream.write(chunk);
-            });
-            inputStream.on("end", resolve);
-            inputStream.on("error", reject);
-        });
-        await fsExtra.remove(fileFullPath);
-    }
-    await fsExtra.remove(uploadDir);
+    (async () => {
+        try {
+            ACTIVE_FINISHED_UPLOADS.add(uploadId);
+            const outputStream = oldFs.createWriteStream(uploadFilePath, { flags: 'a' });
+            for (let i = 1; i <= total; i++) {
+                const fileFullPath = path.join(uploadDir, `${i}-${total}`);
+                const inputStream = oldFs.createReadStream(fileFullPath);
+                await new Promise((resolve, reject) => {
+                    inputStream.on("data", (chunk) => {
+                        outputStream.write(chunk);
+                    });
+                    inputStream.on("end", resolve);
+                    inputStream.on("error", reject);
+                });
+                await fsExtra.remove(fileFullPath);
+            }
+            await fsExtra.remove(uploadDir);
+
+            await onFinished?.(uploadId, files.length);
+        } finally {
+            ACTIVE_FINISHED_UPLOADS.delete(uploadId);
+        }
+    })();
 
-    await onFinished?.(uploadId, files.length);
     return Response.json({ ok: true, finished: true });
 }
 
 export async function processBigFileUpload(astro: AstroGlobal, options: Partial<LoadUploadFilesOptions> = astro.locals.__formsInternalUtils.FORM_OPTIONS.forms?.bigFilesUpload?.bigFileServerOptions) {
     const haveFileUpload = await loadUploadFiles(astro, options);
-    if(haveFileUpload) {
+    if (haveFileUpload) {
         throw new ThrowOverrideResponse(haveFileUpload);
     }
 }