From c2395f63d3894b809c26697041970e68a928f89c Mon Sep 17 00:00:00 2001 From: Sebastian Penhouet Date: Mon, 4 Mar 2024 21:38:37 +0100 Subject: [PATCH] Compute TVL per entity --- src/api/job.ts | 1 + .../precompile_entities_total_value_locked.ts | 128 ++++++++++++++++++ .../precompile_system_total_value_locked.ts | 1 - 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 src/api/workers/precompile_entities_total_value_locked.ts diff --git a/src/api/job.ts b/src/api/job.ts index 220b62f4752..87509555b2d 100644 --- a/src/api/job.ts +++ b/src/api/job.ts @@ -68,6 +68,7 @@ job("API Job", async () => { await Promise.all([ compile(PATHS.systems, "precompile_system_total_value_locked.ts", SYSTEM_RESOURCES), // Depends on "precompile_relations.ts" and "precompile_market_cap.ts" + compile(PATHS.entities, "precompile_entities_total_value_locked.ts", ENTITY_RESOURCES), // Depends on "precompile_relations.ts" and "precompile_market_cap.ts" compile(PATHS.assets, "precompile_collateralization_ratio.ts"), // Depends on "precompile_underlying_assets.ts" ]); diff --git a/src/api/workers/precompile_entities_total_value_locked.ts b/src/api/workers/precompile_entities_total_value_locked.ts new file mode 100644 index 00000000000..9e6f98e4505 --- /dev/null +++ b/src/api/workers/precompile_entities_total_value_locked.ts @@ -0,0 +1,128 @@ +import { hoursToMilliseconds } from "date-fns"; +import { existsSync } from "fs"; +import { sumBy } from "lodash"; +import path from "path"; +import { parentPort } from "worker_threads"; +import { FILES, PATHS } from "../../constants"; +import { ConsecutiveLookup, writeToCsv } from "../../utils/csv"; +import { readJson, remove } from "../../utils/files"; +import { toISOString } from "../../utils/string_formatting"; +import { getDatesBetween } from "../../utils/time"; +import { sendErrorReport } from "../../watcher/bot"; + +interface MarketCapLookup { + assetId: bcked.asset.Id; + lookup: ConsecutiveLookup; +} + +interface MarketCapResult { + assetId: bcked.asset.Id; + marketCap: bcked.asset.MarketCap | undefined; +} + +function initializeMarketCapLookups(assetIds: bcked.asset.Id[]) { + const marketCapLookups: MarketCapLookup[] = []; + + for (const assetId of assetIds) { + const csvPath = path.join(PATHS.assets, assetId, PATHS.records, FILES.csv.marketCap); + + if (!existsSync(csvPath)) { + continue; + } + + marketCapLookups.push({ + assetId, + lookup: new ConsecutiveLookup(csvPath), + }); + } + + return marketCapLookups; +} + +async function getMarketCapForTimestamp( + timestamp: primitive.DateLike, + marketCapLookups: MarketCapLookup[], + window: number = hoursToMilliseconds(12) +): Promise { + // Get closest prices to the current entry for all underlying assets + return Promise.all( + marketCapLookups.map(async ({ assetId, lookup }) => ({ + assetId, + marketCap: await lookup.getClosest(timestamp, window), + })) + ); +} + +async function* computeTotalValueLocked( + id: bcked.entity.Id, + window: number = hoursToMilliseconds(1) +): AsyncIterableIterator { + const assetsJson = path.join(PATHS.entities, id, PATHS.records, FILES.json.assets); + const assets = await readJson<{ ids: bcked.asset.Id[] }>(assetsJson); + + if (!assets) { + console.error("Assets not found"); + return; + } + + const marketCapLookups = initializeMarketCapLookups(assets.ids); + + if (!marketCapLookups.length) { + console.error("No market cap lookups found"); + return; + } + + // TODO get latest entry from total value locked and continue from that time + // const lastEntry = await getLatest(csvPath); + // // Check if the latest records are of the same day. If yes, don't fetch, as there can be no newer data. + // if (lastEntry !== null && isClose(lastEntry.timestamp, Date.now(), hoursInMs(23.99))) return; + + // const startOfRecordings = new Date("2022-11-02"); + // const startDate = new Date(lastEntry?.timestamp ?? startOfRecordings); + const startDate = new Date("2022-11-02"); + + // Loop through the dates using timestamps and create Date objects + for (const timestamp of getDatesBetween(startDate, Date.now(), window)) { + const marketCaps = await getMarketCapForTimestamp(timestamp, marketCapLookups, window); + + if (!marketCaps.length) { + continue; + } + + const totalValueLocked = sumBy(marketCaps, "marketCap.usd"); + + if (!totalValueLocked) { + continue; + } + + yield { + timestamp: toISOString(timestamp), + assets: marketCaps.map((marketCap) => ({ + id: marketCap.assetId, + timestamp: marketCap.marketCap?.timestamp, + })), + totalValueLocked, + }; + } +} + +parentPort?.on("message", async (id: bcked.entity.Id) => { + const step = `Precompile system ${id} total value locked`; + console.log(step); + const filePath = path.join(PATHS.entities, id, PATHS.records, FILES.csv.totalValueLocked); + + try { + // Delete file if it already exists + // TODO Later change this to start at the current date and only append changes + await remove(filePath); + + const entries = computeTotalValueLocked(id); + await writeToCsv(filePath, entries, "timestamp"); + + parentPort?.postMessage(null); + } catch (error) { + console.error(step, error); + await sendErrorReport(step, error); + parentPort?.postMessage(null); + } +}); diff --git a/src/api/workers/precompile_system_total_value_locked.ts b/src/api/workers/precompile_system_total_value_locked.ts index 7bc9bf8c80b..74697d3150c 100644 --- a/src/api/workers/precompile_system_total_value_locked.ts +++ b/src/api/workers/precompile_system_total_value_locked.ts @@ -100,7 +100,6 @@ async function* computeTotalValueLocked( assets: marketCaps.map((marketCap) => ({ id: marketCap.assetId, timestamp: marketCap.marketCap?.timestamp, - usd: marketCap.marketCap?.usd, })), totalValueLocked, };