feat: purge underViews option

example/.microrc.yaml

@@ -30,6 +30,7 @@ restrictFilesToHost: true
 # purge:
 #   overLimit: 1MB # files over this size will be purged
 #   afterTime: 1d # after this many days
+#   underViews: 5 # only delete files with <5 views
 
 # # allowTypes is a list of file types that can be uploaded.
 # # a prefix will be expanded into known types, so `video` becomes `video/mp4`, `video/webm`, etc.

packages/api/src/config.ts

@@ -43,6 +43,7 @@ const schema = strictObject({
   purge: strictObject({
     overLimit: string().transform(parseBytes),
     afterTime: string().transform(ms),
+    underViews: number().optional(),
   }).optional(),
   email: strictObject({
     from: string().email(),

packages/api/src/modules/file/file.service.ts

@@ -1,15 +1,13 @@
 import type { MultipartFile } from "@fastify/multipart";
-import { CreateRequestContext, EntityManager, EntityRepository, MikroORM } from "@mikro-orm/core";
+import { EntityManager, EntityRepository, MikroORM } from "@mikro-orm/core";
 import { InjectRepository } from "@mikro-orm/nestjs";
-import type { OnApplicationBootstrap } from "@nestjs/common";
 import {
   BadRequestException,
   Injectable,
   Logger,
   NotFoundException,
   PayloadTooLargeException,
 } from "@nestjs/common";
-import { Cron, CronExpression } from "@nestjs/schedule";
 import bytes from "bytes";
 import * as contentRange from "content-range";
 import type { FastifyReply, FastifyRequest } from "fastify";
@@ -27,7 +25,7 @@ import type { UserEntity } from "../user/user.entity.js";
 import { FileEntity } from "./file.entity.js";
 
 @Injectable()
-export class FileService implements OnApplicationBootstrap {
+export class FileService {
   @InjectRepository("FileEntity") private fileRepo: EntityRepository<FileEntity>;
 
   private readonly logger = new Logger(FileService.name);
@@ -190,38 +188,4 @@ export class FileService implements OnApplicationBootstrap {
       .header("X-Content-Type-Options", "nosniff")
       .send(stream);
   }
-
-  @Cron(CronExpression.EVERY_HOUR)
-  @CreateRequestContext()
-  async purgeFiles() {
-    if (!config.purge) return;
-    const createdBefore = new Date(Date.now() - config.purge.afterTime);
-    const files = await this.fileRepo.find({
-      size: {
-        $gte: config.purge.overLimit,
-      },
-      createdAt: {
-        $lte: createdBefore,
-      },
-    });
-
-    for (const file of files) {
-      const size = bytes.format(file.size);
-      const age = DateTime.fromJSDate(file.createdAt).toRelative();
-      await this.em.removeAndFlush(file);
-      this.logger.log(`Purging ${file.id} (${size}, ${age})`);
-    }
-
-    if (files[0]) {
-      this.logger.log(`Purged ${files.length} files`);
-    }
-  }
-
-  onApplicationBootstrap() {
-    if (config.purge) {
-      const size = bytes.format(config.purge.overLimit);
-      const age = DateTime.local().minus(config.purge.afterTime).toRelative();
-      this.logger.warn(`Purging files is enabled for files over ${size} uploaded more than ${age}.`);
-    }
-  }
 }

packages/api/src/modules/file/file.subscriber.ts

@@ -16,7 +16,7 @@ export class FileSubscriber implements EventSubscriber<FileEntity> {
     const filesWithHash = await em.count(FileEntity, { hash: entity.hash });
     if (filesWithHash === 0) {
       this.log.debug(`Deleting file on disk with hash ${entity.hash} (${entity.id})`);
-      await this.storageService.delete(entity.hash);
+      await this.storageService.delete(entity);
     }
   }
 }

packages/api/src/modules/storage/storage.service.ts

@@ -1,9 +1,9 @@
-import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import type { EntityRepository } from "@mikro-orm/postgresql";
 import { CreateRequestContext, EntityManager, type FilterQuery } from "@mikro-orm/core";
 import { InjectRepository } from "@mikro-orm/nestjs";
-import { Injectable, Logger } from "@nestjs/common";
-import { Cron, CronExpression, Interval } from "@nestjs/schedule";
+import { Injectable, Logger, type OnApplicationBootstrap } from "@nestjs/common";
+import { Interval } from "@nestjs/schedule";
 import crypto from "crypto";
 import { once } from "events";
 import fs, { existsSync, mkdirSync, rmdirSync } from "fs";
@@ -20,9 +20,11 @@ import { setTimeout } from "timers/promises";
 import ms from "ms";
 import { dedupe } from "../../helpers/dedupe.js";
 import { ThumbnailService } from "../thumbnail/thumbnail.service.js";
+import { DateTime } from "luxon";
+import bytes from "bytes";
 
 @Injectable()
-export class StorageService {
+export class StorageService implements OnApplicationBootstrap {
   @InjectRepository("FileEntity") private fileRepo: EntityRepository<FileEntity>;
 
   private readonly createdPaths = new Set();
@@ -88,16 +90,29 @@ export class StorageService {
     }
   }
 
-  async delete(hash: string) {
+  async delete(file: { hash: string; external: boolean }) {
+    this.logger.debug(`Deleting file "${file.hash}"`);
+    if (file.external) {
+      if (!this.s3Client || !config.externalStorage) {
+        throw new Error("File is external but external storage is not configured");
+      }
+
+      const s3Key = this.getFolderFromHash(file.hash);
+      await this.s3Client.send(
+        new DeleteObjectCommand({
+          Bucket: config.externalStorage.bucket,
+          Key: s3Key,
+        }),
+      );
+
+      return;
+    }
+
     try {
-      this.logger.debug(`Deleting "${hash}" from disk`);
-      const filePath = this.getPathFromHash(hash);
+      const filePath = this.getPathFromHash(file.hash);
       await fs.promises.unlink(filePath);
     } catch (error: any) {
-      if (error.code === "ENOENT") {
-        return;
-      }
-
+      if (error.code === "ENOENT") return;
       throw error;
     }
   }
@@ -150,7 +165,7 @@ export class StorageService {
   @Interval(ms("5s"))
   @CreateRequestContext()
   @dedupe()
-  async uploadToExternalStorage(): Promise<void> {
+  protected async uploadToExternalStorage(): Promise<void> {
     if (!config.externalStorage || !this.s3Client) return;
 
     const filter: FilterQuery<FileEntity>[] = [];
@@ -165,18 +180,23 @@ export class StorageService {
       filter.push({ createdAt: { $lte: decayDate } });
     }
 
-    const file = await this.fileRepo.findOne({
-      external: false,
-      externalError: null,
-      $and: filter,
-      // we want to generate thumbnails while the file is not external, because it
-      // saves s3 calls which may cost money.
-      $or: [
-        { thumbnail: { $ne: null } },
-        { thumbnailError: { $ne: null } },
-        { type: { $nin: [...ThumbnailService.IMAGE_TYPES, ...ThumbnailService.VIDEO_TYPES] } },
-      ],
-    });
+    const file = await this.fileRepo.findOne(
+      {
+        external: false,
+        externalError: null,
+        $and: filter,
+        // we want to generate thumbnails while the file is not external, because it
+        // saves s3 calls which may cost money.
+        $or: [
+          { thumbnail: { $ne: null } },
+          { thumbnailError: { $ne: null } },
+          { type: { $nin: [...ThumbnailService.IMAGE_TYPES, ...ThumbnailService.VIDEO_TYPES] } },
+        ],
+      },
+      {
+        fields: ["type", "size", "hash", "external", "externalError"],
+      },
+    );
 
     if (!file) {
       await setTimeout(ms("5m"));
@@ -212,4 +232,56 @@ export class StorageService {
       await setTimeout(ms("30m"));
     }
   }
+
+  @Interval(ms("5s"))
+  @CreateRequestContext()
+  @dedupe()
+  protected async purgeFiles() {
+    if (!config.purge) return;
+    const createdBefore = new Date(Date.now() - config.purge.afterTime);
+    const filters: FilterQuery<FileEntity> = [];
+    if (config.purge.underViews) {
+      filters.push({
+        views: { $lt: config.purge.underViews },
+      });
+    }
+
+    const files = await this.fileRepo.find(
+      {
+        $and: filters,
+        size: {
+          $gte: config.purge.overLimit,
+        },
+        createdAt: {
+          $lte: createdBefore,
+        },
+      },
+      {
+        limit: 10,
+        fields: ["size", "createdAt", "external", "hash"],
+      },
+    );
+
+    for (const file of files) {
+      const size = bytes.format(file.size);
+      const age = DateTime.fromJSDate(file.createdAt).toRelative();
+      await this.em.removeAndFlush(file);
+      await this.delete(file);
+      this.logger.log(`Purging ${file.id} (${size}, ${age})`);
+    }
+
+    if (files[0]) {
+      this.logger.log(`Purged ${files.length} files`);
+    } else {
+      await setTimeout(ms("5m"));
+    }
+  }
+
+  onApplicationBootstrap() {
+    if (config.purge) {
+      const size = bytes.format(config.purge.overLimit);
+      const age = DateTime.local().minus(config.purge.afterTime).toRelative();
+      this.logger.warn(`Purging files is enabled for files over ${size} uploaded more than ${age}.`);
+    }
+  }
 }

packages/api/src/modules/thumbnail/thumbnail.service.ts

@@ -182,7 +182,7 @@ export class ThumbnailService {
   @Interval(ms("5s"))
   @CreateRequestContext()
   @dedupe()
-  async generateThumbnail(): Promise<void> {
+  protected async generateThumbnail(): Promise<void> {
     const file = await this.fileRepo.findOne({
       thumbnail: null,
       thumbnailError: null,

packages/api/src/orm.config.ts

@@ -1,5 +1,4 @@
 import { FlushMode } from "@mikro-orm/core";
-import type { MikroOrmModuleSyncOptions } from "@mikro-orm/nestjs";
 import { Logger, NotFoundException } from "@nestjs/common";
 import { dirname, join } from "path";
 import { fileURLToPath } from "url";

packages/web/src/renderer/prepass.ts

@@ -1,6 +1,6 @@
-import type { VNode } from "preact";
-import renderToString, { renderToStringAsync } from "preact-render-to-string";
 import type { Client } from "@urql/preact";
+import type { VNode } from "preact";
+import { renderToStringAsync } from "preact-render-to-string";
 
 const MAX_DEPTH = 3;
 const isPromiseLike = (value: unknown): value is Promise<unknown> => {