Add systems tvl to API

src/api/resources/systems.ts

@@ -1,6 +1,126 @@
+import { Stats } from "../../utils/stream";
+import { setDateParts } from "../../utils/time";
 import { icons } from "../utils/icons";
 import { JsonResources } from "../utils/resources";
 
+/**
+ * Converts the statistics object to a summary object.
+ * @param path - The path to set in the summary object, this includes which parts of the date to include e.g. `${path}/{year}/{month}/{day}/{hour}`.
+ * @param stats - The statistics object containing min, max, and median values.
+ * @returns The summary object with low, median, and high values.
+ * @throws Error if the stats object is missing min, max, or median values.
+ */
+function statsToSummary<T extends primitive.Timestamped>(path: string, stats: Stats<T>) {
+    if (!stats.min || !stats.max || !stats.median) {
+        throw new Error("Stats missing. This should have been checked prior.");
+    }
+
+    return {
+        low: {
+            $ref: setDateParts(path, stats.min.timestamp),
+        },
+        median: {
+            $ref: setDateParts(path, stats.median.timestamp),
+        },
+        high: {
+            $ref: setDateParts(path, stats.max.timestamp),
+        },
+    };
+}
+
+function historyResource<T extends primitive.Timestamped>(
+    path: string,
+    latestTimestamp: primitive.ISODateTimeString | undefined,
+    stats: Stats<T> | undefined,
+    years: string[],
+    dateParts: string = "{year}/{month}/{day}/{hour}"
+) {
+    if (!latestTimestamp || !stats || !stats.min || !stats.max || !stats.median || !years.length)
+        return;
+
+    return {
+        $id: path,
+        latest: {
+            $ref: setDateParts(`${path}/${dateParts}`, latestTimestamp),
+        },
+        history: {
+            ...statsToSummary(`${path}/${dateParts}`, stats),
+            data: years.map((year) => ({
+                $ref: `${path}/${year}`,
+            })),
+        },
+    };
+}
+
+function yearResource<T extends primitive.Timestamped>(
+    path: string,
+    stats: Stats<T> | undefined,
+    year: string | undefined,
+    months: string[],
+    dateParts: string = "{year}/{month}/{day}/{hour}"
+) {
+    if (!year || !months.length) return;
+
+    if (!stats?.min || !stats.max || !stats.median) return;
+
+    return {
+        $id: `${path}/${year}`,
+        ...statsToSummary(`${path}/${dateParts}`, stats),
+        data: months.map((month) => ({
+            $ref: `${path}/${year}/${month}`,
+        })),
+    };
+}
+
+function monthResource<T extends primitive.Timestamped>(
+    path: string,
+    stats: Stats<T> | undefined,
+    year: string | undefined,
+    month: string | undefined,
+    days: string[],
+    dateParts: string = "{year}/{month}/{day}/{hour}"
+) {
+    if (!year || !month || !days.length) return;
+
+    if (!stats?.min || !stats.max || !stats.median) return;
+
+    return {
+        $id: `${path}/${year}/${month}`,
+        ...statsToSummary(`${path}/${dateParts}`, stats),
+        data: days.map((day) => ({
+            $ref: `${path}/${year}/${month}/${day}`,
+        })),
+    };
+}
+
+function dayResource<T extends primitive.Timestamped>(
+    path: string,
+    stats: Stats<T> | undefined,
+    year: string | undefined,
+    month: string | undefined,
+    day: string | undefined,
+    hours: string[],
+    dateParts: string = "{year}/{month}/{day}/{hour}"
+) {
+    if (!year || !month || !day || !hours.length) return;
+
+    if (!stats?.min || !stats.max || !stats.median) return;
+
+    return {
+        $id: `${path}/${year}/${month}/${day}`,
+        ...statsToSummary(`${path}/${dateParts}`, stats),
+        data: hours.map((hour) => ({
+            $ref: `${path}/${year}/${month}/${day}/${hour}`,
+        })),
+    };
+}
+
+function hourBaseResource(path: string, timestamp: primitive.ISODateTimeString) {
+    return {
+        $id: setDateParts(`${path}/{year}/{month}/{day}/{hour}`, timestamp),
+        timestamp,
+    };
+}
 class System extends JsonResources {
     constructor() {
         super({
@@ -50,6 +170,9 @@ class System extends JsonResources {
             assets: {
                 $ref: `/systems/${id}/assets`,
             },
+            "total-value-locked": {
+                $ref: `/systems/${id}/total-value-locked`,
+            },
         };
     }
 
@@ -101,6 +224,94 @@ class System extends JsonResources {
             })),
         };
     }
+
+    @JsonResources.register({
+        path: "/systems/{id}/total-value-locked",
+        summary: "Get total value locked of an asset",
+        description: "Get total value locked of an asset by its ID",
+        type: "SystemTotalValueLocked",
+        // TODO write schema
+        schema: {},
+    })
+    async totalValueLockedHistory<T extends primitive.Timestamped>(
+        id: bcked.entity.Id,
+        latestTimestamp: primitive.ISODateTimeString | undefined,
+        stats: Stats<T> | undefined,
+        years: string[]
+    ) {
+        return historyResource(`/systems/${id}/total-value-locked`, latestTimestamp, stats, years);
+    }
+    @JsonResources.register({
+        path: "/systems/{id}/total-value-locked/{year}",
+        summary: "Get total value locked of an asset",
+        description: "Get total value locked of an asset by its ID",
+        type: "SystemTotalValueLocked",
+        // TODO write schema
+        schema: {},
+    })
+    async totalValueLockedYear<T extends primitive.Timestamped>(
+        id: bcked.entity.Id,
+        stats: Stats<T> | undefined,
+        year: string | undefined,
+        months: string[]
+    ) {
+        return yearResource(`/systems/${id}/total-value-locked`, stats, year, months);
+    }
+    @JsonResources.register({
+        path: "/systems/{id}/total-value-locked/{year}/{month}",
+        summary: "Get total value locked of an asset",
+        description: "Get total value locked of an asset by its ID",
+        type: "SystemTotalValueLocked",
+        // TODO write schema
+        schema: {},
+    })
+    async totalValueLockedMonth<T extends primitive.Timestamped>(
+        id: bcked.entity.Id,
+        stats: Stats<T> | undefined,
+        year: string | undefined,
+        month: string | undefined,
+        days: string[]
+    ) {
+        return monthResource(`/systems/${id}/total-value-locked`, stats, year, month, days);
+    }
+    @JsonResources.register({
+        path: "/systems/{id}/total-value-locked/{year}/{month}/{day}",
+        summary: "Get total value locked of an asset",
+        description: "Get total value locked of an asset by its ID",
+        type: "SystemTotalValueLocked",
+        // TODO write schema
+        schema: {},
+    })
+    async totalValueLockedDay<T extends primitive.Timestamped>(
+        id: bcked.entity.Id,
+        stats: Stats<T> | undefined,
+        year: string | undefined,
+        month: string | undefined,
+        day: string | undefined,
+        hours: string[]
+    ) {
+        return dayResource(`/systems/${id}/total-value-locked`, stats, year, month, day, hours);
+    }
+    @JsonResources.register({
+        path: "/systems/{id}/total-value-locked/{year}/{month}/{day}/{hour}",
+        summary: "Get total value locked of an asset",
+        description: "Get total value locked of an asset by its ID",
+        type: "SystemTotalValueLocked",
+        // TODO write schema
+        schema: {},
+    })
+    async totalValueLockedHour(
+        id: bcked.entity.Id,
+        stats: Stats<bcked.entity.TotalValueLocked> | undefined
+    ) {
+        if (!stats?.min || !stats.max || !stats.median) return;
+        return {
+            ...hourBaseResource(`/systems/${id}/total-value-locked`, stats.median.timestamp),
+            value: {
+                "rwa:USD": stats.median.totalValueLocked,
+            },
+        };
+    }
 }
 
 export const SYSTEM_RESOURCES = new System();

src/api/workers/compile_system.ts

@@ -1,9 +1,140 @@
+import { existsSync } from "fs";
+import { PropertyPath } from "lodash";
+import path from "path";
 import { parentPort } from "worker_threads";
-import { PATHS } from "../../constants";
+import { FILES, PATHS } from "../../constants";
+import { readCSV } from "../../utils/csv";
+import { Stats, StreamStats } from "../../utils/stream";
+import { getDateParts } from "../../utils/time";
 import { sendErrorReport } from "../../watcher/bot";
 import { SYSTEM_RESOURCES } from "../resources/systems";
 import { compileAssets, compileDetails, compileIcons } from "../utils/compile";
 
+async function compileHistory<TObject extends primitive.Timestamped, TKey extends keyof TObject>(
+    csvName: string,
+    id: bcked.system.Id,
+    key: TKey | PropertyPath,
+    createHistoryResource: (
+        id: bcked.system.Id,
+        latestTimestamp: primitive.ISODateTimeString | undefined,
+        stats: Stats<TObject> | undefined,
+        years: string[]
+    ) => Promise<any>,
+    createYearResource: (
+        id: bcked.system.Id,
+        stats: Stats<TObject> | undefined,
+        year: string | undefined,
+        months: string[]
+    ) => Promise<any>,
+    createMonthResource: (
+        id: bcked.system.Id,
+        stats: Stats<TObject> | undefined,
+        year: string | undefined,
+        month: string | undefined,
+        days: string[]
+    ) => Promise<any>,
+    createDayResource: (
+        id: bcked.system.Id,
+        stats: Stats<TObject> | undefined,
+        year: string | undefined,
+        month: string | undefined,
+        day: string | undefined,
+        hours: string[]
+    ) => Promise<any>,
+    createHourResource: (id: bcked.system.Id, stats: Stats<TObject> | undefined) => Promise<any>
+) {
+    const csvPath = path.join(PATHS.entities, id, PATHS.records, csvName);
+
+    if (!existsSync(csvPath)) return;
+
+    const historyStats = new StreamStats<TObject, TKey>(key, 100);
+
+    const yearsOfHistory: string[] = [];
+    let yearsStats: StreamStats<TObject, TKey> | undefined;
+    let monthsOfYear: string[] = [];
+    let monthsStats: StreamStats<TObject, TKey> | undefined;
+    let daysOfMonth: string[] = [];
+    let daysStats: StreamStats<TObject, TKey> | undefined;
+    let hoursOfDay: string[] = [];
+    let hoursStats: StreamStats<TObject, TKey> | undefined;
+    let historyObject: TObject | undefined;
+
+    async function addHourToDay(hour: string) {
+        await createHourResource(id, hoursStats?.get());
+        hoursOfDay.push(hour);
+        hoursStats = new StreamStats(key, 100);
+    }
+
+    async function addDayToMonth(day: string, hour: string) {
+        await createDayResource(
+            id,
+            daysStats?.get(),
+            yearsOfHistory.at(-1),
+            monthsOfYear.at(-1),
+            daysOfMonth.at(-1),
+            hoursOfDay
+        );
+        await addHourToDay(hour);
+        hoursOfDay = [hour];
+        daysOfMonth.push(day);
+        daysStats = new StreamStats(key, 100);
+    }
+
+    async function addMonthToYear(month: string, day: string, hour: string) {
+        await createMonthResource(
+            id,
+            monthsStats?.get(),
+            yearsOfHistory.at(-1),
+            monthsOfYear.at(-1),
+            daysOfMonth
+        );
+        await addDayToMonth(day, hour);
+        daysOfMonth = [day];
+        monthsOfYear.push(month);
+        monthsStats = new StreamStats(key, 100);
+    }
+
+    async function addYearToHistory(year: string, month: string, day: string, hour: string) {
+        await createYearResource(id, yearsStats?.get(), yearsOfHistory.at(-1), monthsOfYear);
+        await addMonthToYear(month, day, hour);
+        monthsOfYear = [month];
+        yearsOfHistory.push(year);
+        yearsStats = new StreamStats(key, 100);
+    }
+
+    for await (historyObject of readCSV<TObject>(csvPath)) {
+        const { year, month, day, hour } = getDateParts(historyObject.timestamp);
+
+        if (yearsOfHistory.at(-1) !== year) {
+            await addYearToHistory(year!, month!, day!, hour!);
+        }
+
+        if (monthsOfYear.at(-1) !== month) {
+            await addMonthToYear(month!, day!, hour!);
+        }
+
+        if (daysOfMonth.at(-1) !== day) {
+            await addDayToMonth(day!, hour!);
+        }
+
+        if (hoursOfDay.at(-1) !== hour) {
+            await addHourToDay(hour!);
+        }
+
+        historyStats.add(historyObject);
+        yearsStats!.add(historyObject);
+        monthsStats!.add(historyObject);
+        daysStats!.add(historyObject);
+        hoursStats!.add(historyObject);
+    }
+
+    if (!yearsOfHistory.length) return;
+
+    await createHistoryResource(id, historyObject?.timestamp, historyStats.get(), yearsOfHistory);
+    // Finalize by storing last year, month, day, hour
+    await addYearToHistory("N/A", "N/A", "N/A", "N/A");
+}
+
 parentPort?.on("message", async (id: bcked.system.Id) => {
     console.log(`Compile system ${id}`);
     try {
@@ -12,6 +143,16 @@ parentPort?.on("message", async (id: bcked.system.Id) => {
             compileDetails(SYSTEM_RESOURCES, PATHS.systems, id),
             compileIcons(SYSTEM_RESOURCES, PATHS.systems, id),
             compileAssets(SYSTEM_RESOURCES, PATHS.systems, id),
+            compileHistory<bcked.system.TotalValueLocked, "totalValueLocked">(
+                FILES.csv.totalValueLocked,
+                id,
+                "totalValueLocked",
+                SYSTEM_RESOURCES.totalValueLockedHistory,
+                SYSTEM_RESOURCES.totalValueLockedYear,
+                SYSTEM_RESOURCES.totalValueLockedMonth,
+                SYSTEM_RESOURCES.totalValueLockedDay,
+                SYSTEM_RESOURCES.totalValueLockedHour
+            ),
         ]);
 
         parentPort?.postMessage(null);