refactor: create reusuable functions to fetch logs in batches

Agoric · Nov 22, 2024 · cfa27e6 · cfa27e6
1 parent d26062a
commit cfa27e6
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 50 deletions.
diff --git a/helpers/utils.js b/helpers/utils.js
@@ -111,8 +111,7 @@ export const findEntryWithTimestamp = (logs) => {
   return null;
 };
 
-export const fetchLogsForBatch = async ({
-  type,
+export const fetchLogs = async ({
   network,
   searchQuery,
   startTime,
@@ -127,11 +126,65 @@ export const fetchLogsForBatch = async ({
     resource.labels.namespace_name="${networks[network].namespace_name}" AND
     resource.labels.pod_name="${networks[network].pod_name}" AND
     resource.type="k8s_container" AND
-    jsonPayload.type="${type}" AND
     ${searchQuery} AND
     timestamp >= "${startTime}" AND timestamp <= "${endTime}"
   `;
 
   const [entries] = await logging.getEntries({ filter: queryfilter });
   return entries;
 };
+
+export const calculateDaysDifference = (startTimestamp) => {
+  const startTime = new Date(startTimestamp);
+  const currentDate = new Date();
+
+  console.log(`Calculating days difference...`);
+  console.log(`Start Time: ${startTime.toISOString()}`);
+  console.log(`Current Time: ${currentDate.toISOString()}`);
+
+  const timeDifference = currentDate.getTime() - startTime.getTime();
+  console.log(`Time Difference in Milliseconds: ${timeDifference}`);
+
+  const daysDifference = Math.round(timeDifference / (1000 * 60 * 60 * 24));
+  console.log(`Days since START_BLOCK_EVENT_TYPE: ${daysDifference} days`);
+
+  return daysDifference;
+};
+
+export const fetchLogsInBatches = async ({
+  network,
+  searchQuery,
+  batchSize = 10,
+  totalDaysCoverage = 90,
+}) => {
+  try {
+    let promises = [];
+
+    for (
+      let batchStartIndex = 0;
+      batchStartIndex < totalDaysCoverage;
+      batchStartIndex += batchSize
+    ) {
+      const { startTime, endTime } = getTimestampsForBatch(
+        batchStartIndex,
+        totalDaysCoverage
+      );
+      console.log(`Fetching logs for ${startTime} to ${endTime}`);
+
+      promises.push(
+        fetchLogs({
+          network,
+          searchQuery,
+          startTime,
+          endTime,
+        })
+      );
+    }
+
+    const logs = await Promise.all(promises);
+
+    return logs;
+  } catch (error) {
+    console.error(error);
+  }
+};
diff --git a/services/fetchAndStoreHeightLogs.js b/services/fetchAndStoreHeightLogs.js
@@ -1,31 +1,12 @@
 // @ts-check
 import { fetchGCPLogs } from './fetchGCPLogs.js';
 import {
-  fetchLogsForBatch,
   findEntryWithTimestamp,
-  getTimestampsForBatch,
+  calculateDaysDifference,
+  fetchLogsInBatches,
 } from '../helpers/utils.js';
 import { fs } from 'zx';
 
-const BATCH_SIZE = 10; // 10 days
-
-const calculateDaysDifference = (startTimestamp) => {
-  const startTime = new Date(startTimestamp);
-  const currentDate = new Date();
-
-  console.log(`Calculating days difference...`);
-  console.log(`Start Time: ${startTime.toISOString()}`);
-  console.log(`Current Time: ${currentDate.toISOString()}`);
-
-  const timeDifference = currentDate.getTime() - startTime.getTime();
-  console.log(`Time Difference in Milliseconds: ${timeDifference}`);
-
-  const daysDifference = Math.round(timeDifference / (1000 * 60 * 60 * 24));
-  console.log(`Days since START_BLOCK_EVENT_TYPE: ${daysDifference} days`);
-
-  return daysDifference;
-};
-
 const fetchLogsByBlockEvents = async ({
   network,
   blockHeight,
@@ -35,33 +16,16 @@ const fetchLogsByBlockEvents = async ({
   try {
     console.log(`***** Fetching data for event: ${type} *****`);
 
-    let promises = [];
+    const searchQuery = `
+    jsonPayload.type="${type}" AND
+    jsonPayload.blockHeight="${blockHeight}"`;
 
-    for (
-      let batchStartIndex = 0;
-      batchStartIndex < totalDaysCoverage;
-      batchStartIndex += BATCH_SIZE
-    ) {
-      const { startTime, endTime } = getTimestampsForBatch(
-        batchStartIndex,
-        totalDaysCoverage
-      );
-      console.log(`Fetching logs for ${startTime} to ${endTime}`);
-
-      const searchQuery = `jsonPayload.blockHeight="${blockHeight}"`;
-
-      promises.push(
-        fetchLogsForBatch({
-          network,
-          searchQuery,
-          startTime,
-          endTime,
-          type,
-        })
-      );
-    }
-
-    const logs = await Promise.all(promises);
+    const logs = await fetchLogsInBatches({
+      network,
+      searchQuery,
+      batchSize: 10,
+      totalDaysCoverage,
+    });
 
     return findEntryWithTimestamp(logs);
   } catch (error) {