From c19857e8187738b4523c53603752fa3414f1899a Mon Sep 17 00:00:00 2001 From: rabi-siddique Date: Fri, 22 Nov 2024 16:39:42 +0500 Subject: [PATCH] refactor: eliminate fetchings logs without logging client --- controllers/dateRangeController.js | 12 +--- public/css/styles.css | 33 ++++------ public/index.html | 39 ++++------- public/scripts/submitHeight.js | 38 +++++++++++ services/fetchAndStoreHeightLogs.js | 20 +++--- services/fetchAndStoreLogsFromGCP.js | 21 ++++-- services/fetchGCPLogs.js | 99 ---------------------------- 7 files changed, 88 insertions(+), 174 deletions(-) create mode 100644 public/scripts/submitHeight.js delete mode 100644 services/fetchGCPLogs.js diff --git a/controllers/dateRangeController.js b/controllers/dateRangeController.js index 1fd0513..ae00d96 100644 --- a/controllers/dateRangeController.js +++ b/controllers/dateRangeController.js @@ -45,20 +45,12 @@ export const handleDateRange = async (req, res) => { console.log(`Namespace Name: ${networks[network].namespace_name}`); console.log(`Pod Name: ${networks[network].pod_name}`); - const queryfilter = ` - resource.labels.container_name="${networks[network].container_name}" AND - resource.labels.cluster_name="${networks[network].cluster_name}" AND - resource.labels.namespace_name="${networks[network].namespace_name}" AND - resource.labels.pod_name="${networks[network].pod_name}" AND - resource.type="k8s_container" - `; - - console.log(`Fetching data from GCP for...`); + console.log(`Fetching data from GCP...`); const isSuccessful = await fetchAndStoreLogsFromGCP({ startTime: formattedStartDate, endTime: formattedEndDate, inputFile, - queryfilter, + network, }); if (!isSuccessful) { diff --git a/public/css/styles.css b/public/css/styles.css index b4f5097..87fb199 100644 --- a/public/css/styles.css +++ b/public/css/styles.css @@ -15,7 +15,8 @@ html { border-bottom: 2px solid #ddd; } -.formDateRange { +.formDateRange, +.formBlockHeight { display: flex; flex-direction: column; align-items: start; @@ -47,7 +48,7 @@ html { } } -#submitSearchButton, +#submitHeightButton, #uploadFileButton, #submitDateButton { background-color: #007bff; @@ -95,16 +96,18 @@ select { font-weight: 500; } -#dateForm > #fileHelperText > p { +#dateForm > #fileHelperText > p, +#blockHeightForm > #fileHelperText > p { margin: 0; } -@media (max-width: 599px) { +@media (max-width: 649px) { .topbar { flex-direction: column; gap: 20px; } - .formDateRange { + .formDateRange, + .formBlockHeight { display: block; } @@ -127,22 +130,8 @@ select { #networkForm > #fileHelperText > p { display: none; } -} - -#txHashInput, -#searchTermInput { - display: none; -} -#searchForm { - display: flex; - flex-direction: column-reverse; - gap: 8px; - margin-bottom: 20px; -} - -#searchFormTop, -#searchFormBottom { - display: flex; - gap: 8px; + #blockHeightForm > #fileHelperText > p { + padding-bottom: 8px; + } } diff --git a/public/index.html b/public/index.html index 0099641..bfece7c 100644 --- a/public/index.html +++ b/public/index.html @@ -46,37 +46,17 @@ -
-
- + +
+ + +
- -
-
- - -
- -
- -
- -
- -
- - - +
+

Enter a Block Height

- - -
@@ -84,7 +64,10 @@
- + + + + diff --git a/public/scripts/submitHeight.js b/public/scripts/submitHeight.js new file mode 100644 index 0000000..e260571 --- /dev/null +++ b/public/scripts/submitHeight.js @@ -0,0 +1,38 @@ +document + .getElementById('blockHeightForm') + .addEventListener('submit', async (e) => { + e.preventDefault(); + + const height = document.getElementById('blockHeight').value; + const spinner = document.getElementById('spinnerHeightForm'); + const submitButton = document.getElementById('submitHeightButton'); + const network = document.getElementById('networkSelect').value; + + spinner.style.display = 'inline-block'; + submitButton.style.visibility = 'hidden'; + try { + const response = await fetch('/submit-height', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ height, network }), + }); + + if (response.ok) { + const svgContent = await response.blob(); + const url = URL.createObjectURL(svgContent); + + const svgElement = document.getElementById('svgDisplay'); + svgElement.src = url; + svgElement.style.display = 'inline-block'; + } else { + console.error('Failed to upload file'); + } + } catch (error) { + console.error('Error:', error); + } finally { + spinner.style.display = 'none'; + submitButton.style.visibility = 'visible'; + } + }); diff --git a/services/fetchAndStoreHeightLogs.js b/services/fetchAndStoreHeightLogs.js index e3c556f..1709165 100644 --- a/services/fetchAndStoreHeightLogs.js +++ b/services/fetchAndStoreHeightLogs.js @@ -1,11 +1,12 @@ // @ts-check -import { fetchGCPLogs } from './fetchGCPLogs.js'; import { findEntryWithTimestamp, calculateDaysDifference, fetchLogsInBatches, + fetchLogs, } from '../helpers/utils.js'; import { fs } from 'zx'; +import { ADDITIONAL_QUERY_FILTERS } from '../helpers/constants.js'; const fetchLogsByBlockEvents = async ({ network, @@ -36,7 +37,6 @@ const fetchLogsByBlockEvents = async ({ export const fetchAndStoreHeightLogs = async ({ blockHeight, inputFile, - queryfilter = '', network, }) => { try { @@ -87,19 +87,23 @@ export const fetchAndStoreHeightLogs = async ({ let allEntries = []; - const { entries } = await fetchGCPLogs({ + const searchQuery = ` + ${ADDITIONAL_QUERY_FILTERS} + `; + + const entries = await fetchLogs({ startTime, endTime, - filter: queryfilter, - pageSize: 1000, + searchQuery, + network, }); console.log('Fetched page size: ' + entries.length); allEntries = allEntries.concat(entries); - const logEntries = allEntries.map((entry) => - JSON.stringify(entry.jsonPayload) - ); + const logEntries = allEntries.map((entry) => { + return JSON.stringify(entry.data); + }); if (!logEntries) { throw Error('No Entries found for the given Height'); diff --git a/services/fetchAndStoreLogsFromGCP.js b/services/fetchAndStoreLogsFromGCP.js index 4d726cc..ae68371 100644 --- a/services/fetchAndStoreLogsFromGCP.js +++ b/services/fetchAndStoreLogsFromGCP.js @@ -1,21 +1,28 @@ // @ts-check import { fs } from 'zx'; -import { fetchGCPLogs } from './fetchGCPLogs.js'; +import { fetchLogs } from '../helpers/utils.js'; +import { ADDITIONAL_QUERY_FILTERS } from '../helpers/constants.js'; export const fetchAndStoreLogsFromGCP = async ({ startTime, endTime, inputFile, + network, queryfilter = '', }) => { try { let allEntries = []; - const { entries } = await fetchGCPLogs({ + const searchQuery = ` + ${queryfilter} + ${ADDITIONAL_QUERY_FILTERS} + `; + + const entries = await fetchLogs({ startTime, endTime, - filter: queryfilter, - pageSize: 1000, + searchQuery, + network, }); if (!entries) { @@ -25,9 +32,9 @@ export const fetchAndStoreLogsFromGCP = async ({ console.log('Fetched page size: ' + entries.length); allEntries = allEntries.concat(entries); - const logEntries = allEntries.map((entry) => - JSON.stringify(entry.jsonPayload) - ); + const logEntries = allEntries.map((entry) => { + return JSON.stringify(entry.data); + }); fs.writeFile(inputFile, logEntries.join('\n'), (err) => { if (err) { diff --git a/services/fetchGCPLogs.js b/services/fetchGCPLogs.js deleted file mode 100644 index 83253bf..0000000 --- a/services/fetchGCPLogs.js +++ /dev/null @@ -1,99 +0,0 @@ -// @ts-check -import { getCredentials } from '../helpers/getGCPCredentials.js'; -import { getAccessToken } from '../helpers/getAccessToken.js'; -import { formatDateString } from '../helpers/utils.js'; -import { ADDITIONAL_QUERY_FILTERS } from '../helpers/constants.js'; - -const LOG_ENTRIES_ENDPOINT = 'https://logging.googleapis.com/v2/entries:list'; -// eslint-disable-next-line no-unused-vars -const COMMIT_BLOCK_FINISH_EVENT_TYPE = 'cosmic-swingset-commit-block-finish'; - -/** - * @typedef {{ -* insertId: string; -* jsonPayload: { -* blockHeight: number; -* blockTime: number; -* monotime: number; -* type: typeof COMMIT_BLOCK_FINISH_EVENT_TYPE; -* }; -* labels: { -* 'compute.googleapis.com/resource_name': string; -* 'k8s-pod/app': string; -* 'k8s-pod/apps_kubernetes_io/pod-index': string; -* 'k8s-pod/controller-revision-hash': string; -* 'k8s-pod/grouplb': string; -* 'k8s-pod/statefulset_kubernetes_io/pod-name': string; -* }; -* logName: string; -* receiveTimestamp: string; -* resource: { -* labels: { -* cluster_name: string; -* container_name: string; -* location: string; -* namespace_name: string; -* pod_name: string; -* project_id: string; -* }; -* type: string; -* }; -* severity: string; -* timestamp: string; -* }} LogEntry -* - -/** - * Queries log entries within a specified time range and optional filters. - * - * @param {Object} params - The query parameters. - * @param {string} params.endTime - The end time for the log query in ISO format. - * @param {string} params.startTime - The start time for the log query in ISO format. - * @param {string} [params.filter=''] - Optional filter to narrow down log entries. - * @param {number} [params.pageSize] - Optional number of entries to retrieve per page. - * @param {string} [params.pageToken] - Optional token for pagination. - * @param {boolean} [params.isHeight] - Optional boolean param to update query - * @returns {Promise<{entries: Array, nextPageToken?: string}>} - A promise that resolves with log entries and an optional token for the next page. - */ -export const fetchGCPLogs = async ({ - endTime, - startTime, - filter = '', - pageSize = undefined, - pageToken = undefined, -}) => { - const fullFilter = ` - ${filter} AND - ${ADDITIONAL_QUERY_FILTERS} - timestamp >= "${formatDateString(startTime)}" - AND timestamp <= "${formatDateString(endTime)}" - `; - - const credentials = getCredentials(); - - const body = { - filter: fullFilter, - orderBy: 'timestamp asc', - pageSize, - pageToken, - resourceNames: ['projects/' + credentials.project_id], - }; - - const accessToken = await getAccessToken([ - 'https://www.googleapis.com/auth/logging.read', - ]); - - const response = await fetch(LOG_ENTRIES_ENDPOINT, { - body: JSON.stringify(body), - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - method: 'POST', - }); - - if (!response.ok) - throw Error(`Failed to query logs due to error: ${await response.text()}`); - - return await response.json(); -};