refactor: Change asset storage to be the filesystem instead of sqlite

.gitignore

@@ -38,6 +38,7 @@ next-env.d.ts
 
 # The sqlite database
 **/*.db
+data
 
 # PWA Files
 **/public/sw.js

apps/web/app/api/assets/[assetId]/route.ts

@@ -1,8 +1,6 @@
 import { createContextFromRequest } from "@/server/api/client";
-import { and, eq } from "drizzle-orm";
 
-import { db } from "@hoarder/db";
-import { assets } from "@hoarder/db/schema";
+import { readAsset } from "@hoarder/shared/assetdb";
 
 export const dynamic = "force-dynamic";
 export async function GET(
@@ -13,17 +11,15 @@ export async function GET(
   if (!ctx.user) {
     return Response.json({ error: "Unauthorized" }, { status: 401 });
   }
-  const asset = await db.query.assets.findFirst({
-    where: and(eq(assets.id, params.assetId), eq(assets.userId, ctx.user.id)),
+  const { asset, metadata } = await readAsset({
+    userId: ctx.user.id,
+    assetId: params.assetId,
   });
 
-  if (!asset) {
-    return Response.json({ error: "Asset not found" }, { status: 404 });
-  }
-  return new Response(asset.blob as string, {
+  return new Response(asset, {
     status: 200,
     headers: {
-      "Content-type": asset.contentType,
+      "Content-type": metadata.contentType,
     },
   });
 }

apps/web/app/api/assets/route.ts

@@ -1,8 +1,7 @@
 import { createContextFromRequest } from "@/server/api/client";
 
 import type { ZUploadResponse } from "@hoarder/trpc/types/uploads";
-import { db } from "@hoarder/db";
-import { assets } from "@hoarder/db/schema";
+import { saveAsset } from "@hoarder/shared/assetdb";
 
 const SUPPORTED_ASSET_TYPES = new Set(["image/jpeg", "image/png"]);
 
@@ -34,19 +33,18 @@ export async function POST(request: Request) {
     return Response.json({ error: "Bad request" }, { status: 400 });
   }
 
-  const [dbRes] = await db
-    .insert(assets)
-    .values({
-      encoding: "binary",
-      contentType: contentType,
-      blob: buffer,
-      userId: ctx.user.id,
-    })
-    .returning();
+  const assetId = crypto.randomUUID();
+
+  await saveAsset({
+    userId: ctx.user.id,
+    assetId,
+    metadata: { contentType },
+    asset: buffer,
+  });
 
   return Response.json({
-    assetId: dbRes.id,
-    contentType: dbRes.contentType,
+    assetId,
+    contentType,
     size: buffer.byteLength,
   } satisfies ZUploadResponse);
 }

apps/workers/openaiWorker.ts

@@ -3,12 +3,11 @@ import { and, eq, inArray } from "drizzle-orm";
 import OpenAI from "openai";
 import { z } from "zod";
 
-import Base64 from "js-base64";
-
 import { db } from "@hoarder/db";
-import { assets, bookmarks, bookmarkTags, tagsOnBookmarks } from "@hoarder/db/schema";
+import { bookmarks, bookmarkTags, tagsOnBookmarks } from "@hoarder/db/schema";
 import serverConfig from "@hoarder/shared/config";
 import logger from "@hoarder/shared/logger";
+import { readAsset } from "@hoarder/shared/assetdb";
 import {
   OpenAIQueue,
   queueConnectionDetails,
@@ -142,15 +141,15 @@ async function inferTagsFromImage(
   openai: OpenAI,
 ) {
 
-  const asset = await db.query.assets.findFirst({
-    where: eq(assets.id, bookmark.asset.assetId),
+  const { asset, metadata } = await readAsset({
+    userId: bookmark.userId,
+    assetId: bookmark.asset.assetId,
   });
 
   if (!asset) {
     throw new Error(`[openai][${jobId}] AssetId ${bookmark.asset.assetId} for bookmark ${bookmark.id} not found`);
   }
-
-  const base64 = Base64.encode(asset.blob as string);
+  const base64 = asset.toString('base64');
 
   const chatCompletion = await openai.chat.completions.create({
     model: "gpt-4-vision-preview",
@@ -162,7 +161,7 @@ async function inferTagsFromImage(
           {
             type: "image_url",
             image_url: {
-              url: `data:image/jpeg;base64,${base64}`,
+              url: `data:${metadata.contentType};base64,${base64}`,
               detail: "low",
             },
           },
@@ -176,7 +175,7 @@ async function inferTagsFromImage(
   if (!response) {
     throw new Error(`[openai][${jobId}] Got no message content from OpenAI`);
   }
-  return {response, totalTokens: chatCompletion.usage?.total_tokens};
+  return { response, totalTokens: chatCompletion.usage?.total_tokens };
 }
 
 async function inferTagsFromText(
@@ -194,7 +193,7 @@ async function inferTagsFromText(
   if (!response) {
     throw new Error(`[openai][${jobId}] Got no message content from OpenAI`);
   }
-  return {response, totalTokens: chatCompletion.usage?.total_tokens};
+  return { response, totalTokens: chatCompletion.usage?.total_tokens };
 }
 
 async function inferTags(

apps/workers/package.json

@@ -14,7 +14,6 @@
     "dompurify": "^3.0.9",
     "dotenv": "^16.4.1",
     "drizzle-orm": "^0.29.4",
-    "js-base64": "^3.7.7",
     "jsdom": "^24.0.0",
     "metascraper": "^5.43.4",
     "metascraper-description": "^5.43.4",

packages/db/drizzle/0012_noisy_grim_reaper.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `bookmarkAssets2` (
+	`id` text PRIMARY KEY NOT NULL,
+	`assetType` text NOT NULL,
+	`assetId` text NOT NULL,
+	FOREIGN KEY (`id`) REFERENCES `bookmarks`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+DROP TABLE `assets`;--> statement-breakpoint
+DROP TABLE `bookmarkAssets`;

packages/db/drizzle/0013_square_lady_ursula.sql

@@ -0,0 +1 @@
+ALTER TABLE `bookmarkAssets2` RENAME TO `bookmarkAssets`;