diff --git a/scripts/analytics/usage-report/WorkbookGenerator.ts b/scripts/analytics/usage-report/WorkbookGenerator.ts index 33cceb5d2..84f4f444a 100644 --- a/scripts/analytics/usage-report/WorkbookGenerator.ts +++ b/scripts/analytics/usage-report/WorkbookGenerator.ts @@ -1,4 +1,3 @@ -import EventCode from "@moj-bichard7/common/types/EventCode" import { ReportData, ReportDataResult, reportEventTitles } from "./common" const excel = require("excel4node") @@ -20,11 +19,12 @@ export default class WorkbookGenerator { } private getEventTitle = (eventCode: string) => { - const eventTitlePrefix = eventCode.split(".")[0] === "new-ui" ? "New UI" : "Old UI" + const eventTitlePrefix = + eventCode.split(".")[0] === "new-ui" ? "New: " : eventCode.split(".")[0] === "old-ui" ? "Old: " : "" const actualEventCode = eventCode.split(".").slice(1).join(".") const eventTitle = reportEventTitles[actualEventCode] ?? actualEventCode - return `${eventTitlePrefix} ${eventTitle}` + return `${eventTitlePrefix}${eventTitle}` } public generate = () => { @@ -35,12 +35,8 @@ export default class WorkbookGenerator { this.generateUserReport(this.reportDataResult.allEvents, "All") this.generateDailyUserReport(this.reportDataResult.allEvents, "All") this.generateMonthlyUserReport(this.reportDataResult.allEvents, "All") + this.generateMonthlyForceReport(this.reportDataResult.allEvents, "All") - this.generateDailyReport(this.reportDataResult.withNewUiEvents, "New UI") - this.generateMonthlyReport(this.reportDataResult.withNewUiEvents, "New UI") - this.generateUserReport(this.reportDataResult.withNewUiEvents, "New UI") - this.generateDailyUserReport(this.reportDataResult.withNewUiEvents, "New UI") - this.generateMonthlyUserReport(this.reportDataResult.withNewUiEvents, "New UI") return this } @@ -137,6 +133,26 @@ export default class WorkbookGenerator { this.addFooterRow(sheet, reportData, rowIndex - 2, true) } + private generateMonthlyForceReport = (reportData: ReportData, title: string) => { + const sheet = this.workbook.addWorksheet(`Forces Monthly (${title})`) + this.addHeaderRow(sheet, reportData, "Date", "Force") + + let rowIndex = 2 + Object.entries(reportData.monthlyForces).forEach(([date, forceData]) => { + Object.entries(forceData).forEach(([force, data]) => { + sheet.cell(rowIndex, 1).string(date).style(this.mainCellStyle) + sheet.cell(rowIndex, 2).string(force).style(this.mainCellStyle) + reportData.eventCodes.forEach((eventCode, columnIndex) => { + sheet.cell(rowIndex, columnIndex + 3).number(data[eventCode] || 0) + }) + + rowIndex += 1 + }) + }) + + this.addFooterRow(sheet, reportData, rowIndex - 2, true) + } + private addHeaderRow = (sheet, reportData: ReportData, firstColumnName: string, secondColumnName?: string) => { sheet.cell(1, 1).string(firstColumnName).style(this.mainCellStyle) if (secondColumnName) { @@ -145,8 +161,10 @@ export default class WorkbookGenerator { const startColumnIndex = 2 + (secondColumnName ? 1 : 0) reportData.eventCodes.forEach((eventCode, index) => { - const eventTitle = - (eventCode.split(".")[0] === "new-ui" ? "New UI " : "Old UI ") + eventCode.split(".").slice(1).join() + // REVIEW AND REMOVE + // const eventTitle = + // (eventCode.split(".")[0] === "new-ui" ? "New UI " : eventCode.split(".")[0] === "old-ui" ? "Old UI " : "") + + // eventCode.split(".").slice(1).join() sheet .cell(1, startColumnIndex + index) .string(this.getEventTitle(eventCode)) @@ -174,18 +192,18 @@ export default class WorkbookGenerator { // XX People have the new UI turned on sheet.cell(rowIndex++, 1).string(`${this.usersWithAccessToNewUi.length} People have the new UI turned on`) - Object.entries(reportDataResult.withNewUiEvents.monthly).forEach(([date, newUiMonthlyData]) => { + Object.entries(reportDataResult.allEvents.monthly).forEach(([date, monthlyData]) => { sheet.cell(rowIndex++, 1).string(date).style(this.mainCellStyle) - const newUiMonthlyUsersData = reportDataResult.withNewUiEvents.monthlyUsers[date] + const monthlyUsersData = reportDataResult.allEvents.monthlyUsers[date] // XX People resolved a trigger on the new UI - const totalUsersResolvedTriggersInNewUi = Object.entries(newUiMonthlyUsersData).filter( + const totalUsersResolvedTriggersInNewUi = Object.entries(monthlyUsersData).filter( ([_, monthlyUserData]) => (monthlyUserData["new-ui.triggers.resolved"] || 0) > 0 ).length sheet.cell(rowIndex++, 1).string(`${totalUsersResolvedTriggersInNewUi} People resolved a trigger on the new UI`) // XX People resolved an exception on the new UI - const totalUsersResolvedExceptionsInNewUi = Object.entries(newUiMonthlyUsersData).filter( + const totalUsersResolvedExceptionsInNewUi = Object.entries(monthlyUsersData).filter( ([_, monthlyUserData]) => (monthlyUserData["new-ui.exceptions.resolved"] || 0) > 0 ).length sheet @@ -193,8 +211,8 @@ export default class WorkbookGenerator { .string(`${totalUsersResolvedExceptionsInNewUi} People resolved an exception on the new UI`) // Of XX people who have the UI turned on, XX.XX% of their triggers were resolved in the new UI - const oldUiTriggersResolved = newUiMonthlyData["old-ui.triggers.resolved"] || 0 - const newUiTriggersResolved = newUiMonthlyData["new-ui.triggers.resolved"] || 0 + const oldUiTriggersResolved = monthlyData["old-ui.triggers.resolved"] || 0 + const newUiTriggersResolved = monthlyData["new-ui.triggers.resolved"] || 0 const totalTriggersResolved = oldUiTriggersResolved + newUiTriggersResolved // prettier-ignore const triggersResolvedInNewUiPercentage = (totalTriggersResolved ? newUiTriggersResolved / totalTriggersResolved * 100 : 0 ).toFixed(2) @@ -205,8 +223,8 @@ export default class WorkbookGenerator { ) // Of XX people who have the UI turned on, XX.XX% of their exceptions were resolved in the new UI - const oldUiExceptionResolved = newUiMonthlyData["old-ui.exceptions.resolved"] || 0 - const newUiExceptionResolved = newUiMonthlyData["new-ui.exceptions.resolved"] || 0 + const oldUiExceptionResolved = monthlyData["old-ui.exceptions.resolved"] || 0 + const newUiExceptionResolved = monthlyData["new-ui.exceptions.resolved"] || 0 const totalExceptionsResolved = oldUiExceptionResolved + newUiExceptionResolved // prettier-ignore const exceptionsResolvedInNewUiPercentage = (totalExceptionsResolved ? newUiExceptionResolved / totalExceptionsResolved * 100 : 0).toFixed(2) diff --git a/scripts/analytics/usage-report/common.ts b/scripts/analytics/usage-report/common.ts index 584c6e681..20d64990c 100644 --- a/scripts/analytics/usage-report/common.ts +++ b/scripts/analytics/usage-report/common.ts @@ -1,20 +1,21 @@ -import { AuditLogEvent } from "../../../packages/common/types/AuditLogEvent" import EventCode from "@moj-bichard7/common/types/EventCode" +import { AuditLogEvent } from "../../../packages/common/types/AuditLogEvent" + +type FullAuditLogEvent = AuditLogEvent & { _messageId: string } type ReportData = { - summary: Record + summary: { [key in EventCode]?: number } users: Record> daily: Record> monthly: Record> dailyUsers: Record>> monthlyUsers: Record>> + monthlyForces: Record>> eventCodes: string[] } type ReportDataResult = { allEvents: ReportData - withNewUiEvents: ReportData, - usersWithNewUiEvent: string[] } const reportEventCodes: EventCode[] = [ @@ -22,10 +23,33 @@ const reportEventCodes: EventCode[] = [ EventCode.TriggersLocked, EventCode.TriggersUnlocked, EventCode.TriggersResolved, + EventCode.TriggersGenerated, EventCode.ExceptionsLocked, EventCode.ExceptionsUnlocked, EventCode.ExceptionsResolved, - EventCode.HearingOutcomeReallocated + EventCode.ExceptionsGenerated, + EventCode.HearingOutcomeReallocated, + EventCode.HearingOutcomeResubmittedPhase1, + EventCode.HearingOutcomeResubmittedPhase2, + EventCode.HearingOutcomeDetails, + EventCode.PncResponseNotReceived, + EventCode.PncResponseReceived, + EventCode.IgnoredAlreadyOnPNC, + EventCode.IgnoredAncillary, + EventCode.IgnoredAppeal, + EventCode.IgnoredDisabled, + EventCode.IgnoredNoOffences, + EventCode.IgnoredNonrecordable, + EventCode.IgnoredReopened +] + +const eventCodesToDisplay: (EventCode | string)[] = [ + "Total triggers", + EventCode.TriggersResolved, + "Total exceptions", + EventCode.ExceptionsResolved, + "Resolved exceptions", + "Exceptions resubmitted" ] const reportEventTitles = { @@ -35,9 +59,11 @@ const reportEventTitles = { [EventCode.TriggersResolved]: "Triggers resolved", [EventCode.ExceptionsLocked]: "Exception locked", [EventCode.ExceptionsUnlocked]: "Exception unlocked", - [EventCode.ExceptionsResolved]: "Exception resolved", - [EventCode.HearingOutcomeReallocated]: "Case reallocated" -} as Record + [EventCode.ExceptionsResolved]: "Exception manually resolved", + [EventCode.HearingOutcomeReallocated]: "Case reallocated", + [EventCode.HearingOutcomeResubmittedPhase1]: "Resubmitted to Phase 1", + [EventCode.HearingOutcomeResubmittedPhase2]: "Resubmitted to Phase 2" +} as Record const log = (...params: unknown[]) => { const logContent = [new Date().toISOString(), " - ", ...params] @@ -48,4 +74,14 @@ const getDateString = (date: string | Date) => (typeof date === "object" ? date. const isNewUIEvent = (event: AuditLogEvent) => event.eventSource === "Bichard New UI" -export { reportEventCodes, reportEventTitles, ReportData, ReportDataResult, getDateString, log, isNewUIEvent } +export { + eventCodesToDisplay, + FullAuditLogEvent, + getDateString, + isNewUIEvent, + log, + ReportData, + ReportDataResult, + reportEventCodes, + reportEventTitles +} diff --git a/scripts/analytics/usage-report/fetchEvents.ts b/scripts/analytics/usage-report/fetchEvents.ts index a51ab1c39..0ea39c5f0 100644 --- a/scripts/analytics/usage-report/fetchEvents.ts +++ b/scripts/analytics/usage-report/fetchEvents.ts @@ -1,17 +1,25 @@ -import { AuditLogEvent } from "../../../packages/common/types/AuditLogEvent" -import { DocumentClient } from "aws-sdk/clients/dynamodb" import { isError } from "@moj-bichard7/common/types/Result" -import { reportEventCodes, getDateString, isNewUIEvent, log } from "./common" +import { DocumentClient } from "aws-sdk/clients/dynamodb" +import { FullAuditLogEvent, getDateString, isNewUIEvent, log, reportEventCodes } from "./common" + +const generateDates = (start: Date, end: Date): Date[] => { + const dates: Date[] = [] + let currentDate = new Date(start) + while (currentDate < end) { + dates.push(new Date(currentDate)) + currentDate.setDate(currentDate.getDate() + 1) + } + + return dates +} -const filterEvents = (events: AuditLogEvent[]): AuditLogEvent[] => +const filterEvents = (events: FullAuditLogEvent[]): FullAuditLogEvent[] => events.filter((event) => reportEventCodes.includes(event.eventCode)) -const findEvents = async (dynamo: DocumentClient, eventsTableName: string, start: Date, end: Date) => { - log(`Getting messages for the period between ${getDateString(start)} and ${getDateString(end)}`) +const fetchEvents = async (dynamo: DocumentClient, eventsTableName: string, start: Date, end: Date) => { let lastEvaluatedKey - let events: AuditLogEvent[] = [] const messageIds = new Set() - console.log("Fetching messages and events...") + let events: FullAuditLogEvent[] = [] while (true) { const query: DocumentClient.QueryInput = { @@ -40,13 +48,13 @@ const findEvents = async (dynamo: DocumentClient, eventsTableName: string, start return eventsResult } - if (eventsResult.Items.length === 0) { + if (!eventsResult.Items || eventsResult.Items.length === 0) { return events } lastEvaluatedKey = eventsResult?.LastEvaluatedKey - let fetchedEvents = (eventsResult.Items ?? []) as (AuditLogEvent & { _messageId: string })[] - fetchedEvents = filterEvents(fetchedEvents) as (AuditLogEvent & { _messageId: string })[] + let fetchedEvents = (eventsResult.Items ?? []) as FullAuditLogEvent[] + fetchedEvents = filterEvents(fetchedEvents) as FullAuditLogEvent[] fetchedEvents.forEach((event) => messageIds.add(event._messageId)) events = events.concat(filterEvents(fetchedEvents)) @@ -69,4 +77,52 @@ const findEvents = async (dynamo: DocumentClient, eventsTableName: string, start } } +const findEvents = async ( + dynamo: DocumentClient, + eventsTableName: string, + start: Date, + end: Date +): Promise => { + log(`Getting messages for the period between ${getDateString(start)} and ${getDateString(end)}`) + const allEvents: FullAuditLogEvent[] = [] + console.log(`Fetching messages and events between ${start.toISOString()} and ${end.toISOString()}...`) + const dates = generateDates(start, end) + const totalDates = dates.length + + const worker = async () => { + while (dates.length > 0) { + const date = dates.shift() + if (!date) { + break + } + + const endDate = new Date(date) + endDate.setDate(endDate.getDate() + 1) + const events = await fetchEvents(dynamo, eventsTableName, date, endDate) + if (isError(events)) { + throw events + } + + allEvents.push(...events) + } + } + + const reporter = async () => { + while (dates.length > 0) { + console.log(`Fetch events for dates ${totalDates - dates.length} of ${totalDates}`) + + await new Promise((resolve) => setTimeout(resolve, 3000)) + } + } + + await Promise.all( + new Array(10) + .fill(0) + .map(() => worker()) + .concat(reporter()) + ) + + return allEvents.sort((a, b) => (a.timestamp > b.timestamp ? 1 : -1)) +} + export default findEvents diff --git a/scripts/analytics/usage-report/fetchForceOwners.ts b/scripts/analytics/usage-report/fetchForceOwners.ts new file mode 100644 index 000000000..c10b96abe --- /dev/null +++ b/scripts/analytics/usage-report/fetchForceOwners.ts @@ -0,0 +1,69 @@ +import EventCode from "@moj-bichard7/common/types/EventCode" +import { isError } from "@moj-bichard7/common/types/Result" +import { DocumentClient } from "aws-sdk/clients/dynamodb" +import { FullAuditLogEvent } from "./common" +import getForceOwner from "./getForceOwner" + +const fetchForceOwners = async ( + events: FullAuditLogEvent[], + dynamo: DocumentClient, + auditLogTableName: string +): Promise> => { + const messageIdsSet = events.reduce((acc, event) => { + acc.add(event._messageId) + return acc + }, new Set()) + + const messageIds = Array.from(messageIdsSet) + const forceOwners: Record = {} + events + .filter((event) => event.eventCode === EventCode.HearingOutcomeDetails) + .forEach((event) => { + const forceOwner = event.attributes?.["Force Owner"]?.toString().substring(0, 2) as string + if (forceOwner) { + forceOwners[event._messageId] = Number(forceOwner) + } + }) + const totalMessages = messageIds.length + + const worker = async () => { + while (messageIds.length > 0) { + const messageId = messageIds.shift() + if (!messageId) { + break + } + + if (forceOwners[messageId]) { + continue + } + + const forceOwner = await getForceOwner(dynamo, auditLogTableName, messageId) + if (isError(forceOwner)) { + throw forceOwner + } + + forceOwners[messageId] = Number(forceOwner) + } + } + + const reporter = async () => { + while (messageIds.length > 0) { + console.log(`Fetched force owner for ${totalMessages - messageIds.length} of ${totalMessages}`) + + await new Promise((resolve) => setTimeout(resolve, 3000)) + } + } + + await Promise.all( + new Array(50) + .fill(0) + .map(() => worker()) + .concat(reporter()) + ) + + console.log(`Fetched force owner for ${totalMessages - messageIds.length} of ${totalMessages}`) + + return forceOwners +} + +export default fetchForceOwners diff --git a/scripts/analytics/usage-report/generateReportData.ts b/scripts/analytics/usage-report/generateReportData.ts index bcf26ba2d..f166bea70 100644 --- a/scripts/analytics/usage-report/generateReportData.ts +++ b/scripts/analytics/usage-report/generateReportData.ts @@ -1,6 +1,42 @@ import EventCode from "@moj-bichard7/common/types/EventCode" -import { AuditLogEvent } from "../../../packages/common/types/AuditLogEvent" -import { ReportDataResult, getDateString, isNewUIEvent } from "./common" +import { FullAuditLogEvent, ReportDataResult, eventCodesToDisplay, getDateString, isNewUIEvent } from "./common" + +type UpdateReportDataOptions = { + event: FullAuditLogEvent + eventCodeKey: string + numberToAdd: number +} + +const editableExceptions = [ + "HO100102", + "HO100323", + "HO100200", + "HO100300", + "HO100322", + "HO100206", + "HO100301", + "HO100321", + "H0100329", + "H0100332" +] + +const AUTO_RESUBMISSION_EXCEPTIONS = ["HO100314", "HO100302", "HO100404", "HO100402"] + +const eventsPostExceptionsResolved = [ + EventCode.PncUpdated, + EventCode.IgnoredAlreadyOnPNC, + EventCode.IgnoredAncillary, + EventCode.IgnoredAppeal, + EventCode.IgnoredDisabled, + EventCode.IgnoredNoOffences, + EventCode.IgnoredNonrecordable, + EventCode.IgnoredReopened +] + +const currentExceptions: Record = {} +const currentResubmissionUi: Record = {} +const exceptionsResolvedManually = EventCode.ExceptionsResolved +const exceptionsResubmittedEvents = [EventCode.HearingOutcomeResubmittedPhase1, EventCode.HearingOutcomeResubmittedPhase2] const getDates = (start: Date, end: Date) => { let date = new Date(start) @@ -13,142 +49,229 @@ const getDates = (start: Date, end: Date) => { return dates } -const getEventCodeKey = (event: AuditLogEvent) => (isNewUIEvent(event) ? "new-ui." : "old-ui.") + event.eventCode +const getEventCodeKey = (event: FullAuditLogEvent) => (isNewUIEvent(event) ? "new-ui." : "old-ui.") + event.eventCode -const generateReportData = (events: AuditLogEvent[], start: Date, end: Date): ReportDataResult => { - const allEventCodes = new Set() - const newUiEventCodes = new Set() - const usersWithNewUiEvent = new Set() +const getResolvedExceptions = (event: FullAuditLogEvent) => { + if (event.eventCode !== EventCode.ExceptionsGenerated) { + return currentExceptions[event._messageId] + } - const allDailyData = {} as Record> - const allMonthlyData = {} as Record> - const allDailyUserData = {} as Record>> - const allMonthlyUserData = {} as Record>> - const allUserData = {} as Record> - const allSummary = {} as Record + const previousExceptions = currentExceptions[event._messageId] + const newExceptions = extractExceptionCodes(event) + return previousExceptions.filter((previousException) => !newExceptions.includes(previousException)) +} - const newUiDailyData = {} as Record> - const newUiMonthlyData = {} as Record> - const newUiDailyUserData = {} as Record>> - const newUiMonthlyUserData = {} as Record>> - const newUiUserData = {} as Record> - const newUiSummary = {} as Record +const extractExceptionCodes = (event: FullAuditLogEvent): string[] => + Object.keys(event.attributes ?? {}) + .filter((key) => /Error \d{1,2} Details/.test(key)) + .map((key) => event.attributes?.[key]?.toString().split("||")[0]) as string[] - getDates(start, end).forEach((date: string) => { - const month = date.substring(0, 7) - allDailyData[date] = {} - allMonthlyData[month] = {} - newUiDailyData[date] = {} - newUiMonthlyData[month] = {} - }) +const excludeAutoResubmissionExceptions = (exceptionCode: string): boolean => + !AUTO_RESUBMISSION_EXCEPTIONS.includes(exceptionCode) - events.forEach((event) => { - const date = getDateString(event.timestamp) - const month = date.substring(0, 7) - const eventCodeKey = getEventCodeKey(event) - const username = event.user ?? "unknown" - allEventCodes.add(eventCodeKey) - if (isNewUIEvent(event)) { - usersWithNewUiEvent.add(username) - } +const filterAutoResubmissionExceptions = (exceptionCode: string): boolean => + AUTO_RESUBMISSION_EXCEPTIONS.includes(exceptionCode) - allSummary[eventCodeKey] = (allSummary[eventCodeKey] || 0) + 1 +const createUpdateReportDataFunction = + (forceOwners: Record, date: string, month: string) => + async (reportData: any, { event, eventCodeKey, numberToAdd }: UpdateReportDataOptions) => { + reportData.allSummary[eventCodeKey] = (reportData.allSummary[eventCodeKey] || 0) + numberToAdd - allUserData[username] = { - ...(allUserData[username] ?? {}), - [eventCodeKey]: (allUserData[username]?.[eventCodeKey] || 0) + 1 + reportData.allDailyData[date] = { + ...(reportData.allDailyData[date] ?? {}), + [eventCodeKey]: (reportData.allDailyData[date]?.[eventCodeKey] || 0) + numberToAdd } - allDailyData[date] = { - ...(allDailyData[date] ?? {}), - [eventCodeKey]: (allDailyData[date]?.[eventCodeKey] || 0) + 1 + reportData.allMonthlyData[month] = { + ...(reportData.allMonthlyData[month] ?? {}), + [eventCodeKey]: (reportData.allMonthlyData[month]?.[eventCodeKey] || 0) + numberToAdd } - allMonthlyData[month] = { - ...(allMonthlyData[month] ?? {}), - [eventCodeKey]: (allMonthlyData[month]?.[eventCodeKey] || 0) + 1 - } + if (event.eventCode !== EventCode.ExceptionsGenerated) { + let username = event.user ?? "unknown" + reportData.allUserData[username] = { + ...(reportData.allUserData[username] ?? {}), + [eventCodeKey]: (reportData.allUserData[username]?.[eventCodeKey] || 0) + numberToAdd + } - allDailyUserData[date] = { - ...(allDailyUserData[date] ?? {}), - [username]: { - ...(allDailyUserData[date]?.[username] ?? {}), - [eventCodeKey]: (allDailyUserData[date]?.[username]?.[eventCodeKey] || 0) + 1 + reportData.allDailyUserData[date] = { + ...(reportData.allDailyUserData[date] ?? {}), + [username]: { + ...(reportData.allDailyUserData[date]?.[username] ?? {}), + [eventCodeKey]: (reportData.allDailyUserData[date]?.[username]?.[eventCodeKey] || 0) + numberToAdd + } + } + + reportData.allMonthlyUserData[month] = { + ...(reportData.allMonthlyUserData[month] ?? {}), + [username]: { + ...(reportData.allMonthlyUserData[month]?.[username] ?? {}), + [eventCodeKey]: (reportData.allMonthlyUserData[month]?.[username]?.[eventCodeKey] || 0) + numberToAdd + } } } - allMonthlyUserData[month] = { - ...(allMonthlyUserData[month] ?? {}), - [username]: { - ...(allMonthlyUserData[month]?.[username] ?? {}), - [eventCodeKey]: (allMonthlyUserData[month]?.[username]?.[eventCodeKey] || 0) + 1 + const forceOwner = forceOwners[event._messageId] + reportData.allMonthlyForceData[month] = { + ...(reportData.allMonthlyForceData[month] ?? {}), + [forceOwner!]: { + ...(reportData.allMonthlyForceData[month]?.[forceOwner!] ?? {}), + [eventCodeKey]: (reportData.allMonthlyForceData[month]?.[forceOwner!]?.[eventCodeKey] || 0) + numberToAdd } } - }) + } - events - .filter((event) => usersWithNewUiEvent.has(event.user ?? "Unknown")) - .forEach((event) => { - const date = getDateString(event.timestamp) - const month = date.substring(0, 7) - const eventCodeKey = getEventCodeKey(event) - const username = event.user ?? "unknown" - newUiEventCodes.add(eventCodeKey) +const generateReportData = async ( + events: FullAuditLogEvent[], + start: Date, + end: Date, + forceOwners: Record +): Promise => { + const reportData = { + allEventCodes: new Set(), + newUiEventCodes: new Set(), + usersWithNewUiEvent: new Set(), - newUiSummary[eventCodeKey] = (newUiSummary[eventCodeKey] || 0) + 1 + allDailyData: {} as Record>, + allMonthlyData: {} as Record>, + allDailyUserData: {} as Record>>, + allMonthlyUserData: {} as Record>>, + allMonthlyForceData: {} as Record>>, + allUserData: {} as Record>, + allSummary: {} as Record, + + newUiDailyData: {} as Record>, + newUiMonthlyData: {} as Record>, + newUiDailyUserData: {} as Record>>, + newUiMonthlyUserData: {} as Record>>, + newUiUserData: {} as Record>, + newUiSummary: {} as Record + } + + getDates(start, end).forEach((date: string) => { + const month = date.substring(0, 7) + reportData.allDailyData[date] = {} + reportData.allMonthlyData[month] = {} + reportData.newUiDailyData[date] = {} + reportData.newUiMonthlyData[month] = {} + }) - newUiUserData[username] = { - ...(newUiUserData[username] ?? {}), - [eventCodeKey]: (newUiUserData[username]?.[eventCodeKey] || 0) + 1 + for (const event of events) { + if (event.eventSource === "ResubmitFailedPNCMessages") { + if (currentExceptions[event._messageId]) { + delete currentExceptions[event._messageId] } + continue + } - newUiDailyData[date] = { - ...(newUiDailyData[date] ?? {}), - [eventCodeKey]: (newUiDailyData[date]?.[eventCodeKey] || 0) + 1 + if (event.eventCode === EventCode.HearingOutcomeDetails) { + const forceOwner = event.attributes?.["Force Owner"]?.toString()?.substring(0, 2) + if (forceOwner) { + forceOwners[event._messageId] = Number(forceOwner) } - newUiMonthlyData[month] = { - ...(newUiMonthlyData[month] ?? {}), - [eventCodeKey]: (newUiMonthlyData[month]?.[eventCodeKey] || 0) + 1 + continue + } + + const date = getDateString(event.timestamp) + const month = date.substring(0, 7) + const updateReportData = createUpdateReportDataFunction(forceOwners, date, month) + + let eventCodeKey = getEventCodeKey(event) + let numberToAdd = 1 + + if ( + currentExceptions[event._messageId] && + currentResubmissionUi[event._messageId] && + (event.eventCode === EventCode.ExceptionsGenerated || eventsPostExceptionsResolved.includes(event.eventCode)) + ) { + const customEventCodeKey = `${currentResubmissionUi[event._messageId]}.Resolved exceptions` + reportData.allEventCodes.add(customEventCodeKey) + const resolvedExceptions = getResolvedExceptions(event) + + await updateReportData(reportData, { + event, + numberToAdd: resolvedExceptions.length, + eventCodeKey: customEventCodeKey + }) + + // Editable exceptions + const resolvedEditableExceptions = resolvedExceptions.filter((exceptionCode) => + editableExceptions.includes(exceptionCode) + ) + if (resolvedEditableExceptions.length > 0) { + numberToAdd = resolvedEditableExceptions.length + const customEventCodeKey = `${currentResubmissionUi[event._messageId]}.Resolved editable exceptions` + reportData.allEventCodes.add(customEventCodeKey) + await updateReportData(reportData, { + event, + numberToAdd, + eventCodeKey: customEventCodeKey + }) } + } - newUiDailyUserData[date] = { - ...(newUiDailyUserData[date] ?? {}), - [username]: { - ...(newUiDailyUserData[date]?.[username] ?? {}), - [eventCodeKey]: (newUiDailyUserData[date]?.[username]?.[eventCodeKey] || 0) + 1 - } + if (event.eventCode === EventCode.ExceptionsGenerated) { + eventCodeKey = ".Total exceptions" + currentExceptions[event._messageId] = extractExceptionCodes(event) + const numberOfExceptions = currentExceptions[event._messageId].filter(excludeAutoResubmissionExceptions).length + numberToAdd = numberOfExceptions + } + + if (event.eventCode === exceptionsResolvedManually) { + const autoResubmissionExceptions = + currentExceptions[event._messageId]?.filter(filterAutoResubmissionExceptions).length || 0 + + if (autoResubmissionExceptions > 0) { + await updateReportData(reportData, { + event, + numberToAdd: autoResubmissionExceptions, + eventCodeKey: ".Total exceptions" + }) } + numberToAdd = currentExceptions[event._messageId]?.length || 0 + } - newUiMonthlyUserData[month] = { - ...(newUiMonthlyUserData[month] ?? {}), - [username]: { - ...(newUiMonthlyUserData[month]?.[username] ?? {}), - [eventCodeKey]: (newUiMonthlyUserData[month]?.[username]?.[eventCodeKey] || 0) + 1 - } + if (event.eventCode === EventCode.TriggersGenerated) { + numberToAdd = Number(event?.attributes?.["Number of Triggers"] || 0) + eventCodeKey = ".Total triggers" + } + + if (event.eventCode === EventCode.TriggersResolved) { + numberToAdd = Number(event?.attributes?.["Number Of Triggers"] || 0) + } + + if (exceptionsResubmittedEvents.includes(event.eventCode)) { + eventCodeKey = (isNewUIEvent(event) ? "new-ui" : "old-ui") + ".Exceptions resubmitted" + currentResubmissionUi[event._messageId] = isNewUIEvent(event) ? "new-ui" : "old-ui" + numberToAdd = currentExceptions[event._messageId]?.length ?? 0 + if (event.user === "System") { + numberToAdd = 0 } - }) + } + + reportData.allEventCodes.add(eventCodeKey) + + await updateReportData(reportData, { event, numberToAdd, eventCodeKey }) + } + + reportData.allEventCodes = new Set( + Array.from(reportData.allEventCodes).filter((eventCode) => + eventCodesToDisplay.some((eventCodeToDisplay) => eventCode.includes(eventCodeToDisplay)) + ) + ) return { allEvents: { - summary: allSummary, - users: allUserData, - daily: allDailyData, - monthly: allMonthlyData, - dailyUsers: allDailyUserData, - monthlyUsers: allMonthlyUserData, - eventCodes: Array.from(allEventCodes) - }, - withNewUiEvents: { - summary: newUiSummary, - users: newUiUserData, - daily: newUiDailyData, - monthly: newUiMonthlyData, - dailyUsers: newUiDailyUserData, - monthlyUsers: newUiMonthlyUserData, - eventCodes: Array.from(newUiEventCodes) - }, - usersWithNewUiEvent: Array.from(usersWithNewUiEvent) + summary: reportData.allSummary, + users: reportData.allUserData, + daily: reportData.allDailyData, + monthly: reportData.allMonthlyData, + dailyUsers: reportData.allDailyUserData, + monthlyUsers: reportData.allMonthlyUserData, + monthlyForces: reportData.allMonthlyForceData, + eventCodes: Array.from(reportData.allEventCodes) + } } } diff --git a/scripts/analytics/usage-report/getForceOwner.ts b/scripts/analytics/usage-report/getForceOwner.ts new file mode 100644 index 000000000..85b095590 --- /dev/null +++ b/scripts/analytics/usage-report/getForceOwner.ts @@ -0,0 +1,27 @@ +import { isError } from "@moj-bichard7/common/types/Result" +import { DocumentClient } from "aws-sdk/clients/dynamodb" +// import { DynamoDB } from "@aws-sdk/client-dynamodb"; + +const getForceOwner = async (dynamo: DocumentClient, auditLogTableName: string, messageId: string): Promise => { + const query: DocumentClient.GetItemInput = { + TableName: auditLogTableName, + Key: { + messageId + }, + ProjectionExpression: "forceOwner", + ConsistentRead: false + } + + const auditLogResult = await dynamo + .get(query) + .promise() + .catch((error: Error) => error) + + if (isError(auditLogResult)) { + return auditLogResult + } + + return auditLogResult.Item?.forceOwner +} + +export default getForceOwner diff --git a/scripts/analytics/usage-report/index.ts b/scripts/analytics/usage-report/index.ts index 3f2e73344..17a2e5cb5 100644 --- a/scripts/analytics/usage-report/index.ts +++ b/scripts/analytics/usage-report/index.ts @@ -13,17 +13,17 @@ * */ -import { Lambda, RDS } from "aws-sdk" -import { DynamoDB } from "aws-sdk" -import { DocumentClient } from "aws-sdk/clients/dynamodb" +import baseConfig from "@moj-bichard7/common/db/baseConfig" import { isError } from "@moj-bichard7/common/types/Result" +import { DynamoDB, Lambda, RDS } from "aws-sdk" +import { DocumentClient } from "aws-sdk/clients/dynamodb" +import { DataSource } from "typeorm" +import { getDateString } from "./common" import findEvents from "./fetchEvents" +import { findUsersWithAccessToNewUi } from "./findUsersWithAccessToNewUi" import generateReportData from "./generateReportData" -import { getDateString } from "./common" import WorkbookGenerator from "./WorkbookGenerator" -import { DataSource } from "typeorm" -import { findUsersWithAccessToNewUi } from "./findUsersWithAccessToNewUi" -import baseConfig from "@moj-bichard7/common/db/baseConfig" +import fetchForceOwners from "./fetchForceOwners" const WORKSPACE = process.env.WORKSPACE ?? "production" let dynamo: DocumentClient @@ -66,8 +66,8 @@ async function setup() { throw Error("Couldn't get Postgres connection details (describeDBInstances)") } - const dbHost = dbInstances.DBClusters?.map((clusters) => clusters.ReaderEndpoint).filter( - (endpoint) => endpoint?.startsWith(`cjse-${WORKSPACE}-bichard-7-aurora-cluster.cluster-ro-`) + const dbHost = dbInstances.DBClusters?.map((clusters) => clusters.ReaderEndpoint).filter((endpoint) => + endpoint?.startsWith(`cjse-${WORKSPACE}-bichard-7-aurora-cluster.cluster-ro-`) )?.[0] process.env.DB_USER = process.env.DB_PASSWORD = process.env.DB_SSL = "true" postgres = await new DataSource({ @@ -97,8 +97,12 @@ const run = async () => { throw events } + console.log("Fetching force owners...") + const auditLogTableName = "bichard-7-production-audit-log" + const forceOwners = await fetchForceOwners(events, dynamo, auditLogTableName) + console.log("Generating report data...") - const reportData = generateReportData(events, start, end) + const reportData = await generateReportData(events, start, end, forceOwners) console.log("Generating report workbook...") const reportFilename = `New UI Report (${getDateString(start)} to ${getDateString(end)}).xlsx`