feat: Remove BFJ for async streaming (#227)

* Remove BFJ for async streaming

- BFJ has been archived repository for some time now
- Its use has been limited as writeErrorLogFile loads the
  errorLog completely into memory, transforms it, then writes
  it to a log file using bfj. Obviously if the data is so large
  that it requires bfj, then loading it completely into to
  memory would more than likely crash the process.
- BFJ does not have @types/bfj available

* Apply lint fixes

* Fix rename issue

* Improve expectations on writeErrorLogFile tests

* Fix mocking createWriteStream in tests

* Remove bfj from dependabot

* Revert to using Writeable from node:stream for mocking createWriteStream

Author: Brad-Turner

.github/dependabot.yml

@@ -9,9 +9,6 @@ updates:
       timezone: UTC
     open-pull-requests-limit: 10
     ignore:
-      - dependency-name: bfj # ignore ESM only versions
-        versions:
-          - ">=8.0.0"
       - dependency-name: figures  # ignore ESM only versions
         versions:
           - ">=4.0.0"

lib/logging.js

@@ -1,10 +1,10 @@
-import EventEmitter from 'events'
-
-import bfj from 'bfj'
-import figures from 'figures'
 import format from 'date-fns/format'
 import parseISO from 'date-fns/parseISO'
-
+import figures from 'figures'
+import EventEmitter from 'node:events'
+import { createWriteStream } from 'node:fs'
+import { Readable, Transform } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
 import getEntityName from './get-entity-name'
 
 export const logEmitter = new EventEmitter()
@@ -47,7 +47,9 @@ export function formatLogMessageOneLine (logMessage) {
       errorOutput.push(`Entity: ${getEntityName(data.entity)}`)
     }
     if ('details' in data && 'errors' in data.details) {
-      const errorList = data.details.errors.map((error) => error.details || error.name)
+      const errorList = data.details.errors.map(
+        (error) => error.details || error.name
+      )
       errorOutput.push(`Details: ${errorList.join(', ')}`)
     }
     if ('requestId' in data) {
@@ -78,7 +80,9 @@ export function formatLogMessageLogfile (logMessage) {
   } catch (err) {
     // Fallback for errors without API information
     if (logMessage.error.stack) {
-      logMessage.error.stacktrace = logMessage.error.stack.toString().split(/\n +at /)
+      logMessage.error.stacktrace = logMessage.error.stack
+        .toString()
+        .split(/\n +at /)
     }
   }
 
@@ -94,47 +98,78 @@ export function formatLogMessageLogfile (logMessage) {
 // Display all errors
 export function displayErrorLog (errorLog) {
   if (errorLog.length) {
-    const count = errorLog.reduce((count, curr) => {
-      if (Object.prototype.hasOwnProperty.call(curr, 'warning')) count.warnings++
-      else if (Object.prototype.hasOwnProperty.call(curr, 'error')) count.errors++
-      return count
-    }, { warnings: 0, errors: 0 })
-
-    console.log(`\n\nThe following ${count.errors} errors and ${count.warnings} warnings occurred:\n`)
+    const count = errorLog.reduce(
+      (count, curr) => {
+        if (Object.prototype.hasOwnProperty.call(curr, 'warning')) {
+          count.warnings++
+        } else if (Object.prototype.hasOwnProperty.call(curr, 'error')) {
+          count.errors++
+        }
+        return count
+      },
+      { warnings: 0, errors: 0 }
+    )
+
+    console.log(
+      `\n\nThe following ${count.errors} errors and ${count.warnings} warnings occurred:\n`
+    )
 
     errorLog
-      .map((logMessage) => `${format(parseISO(logMessage.ts), 'HH:mm:ss')} - ${formatLogMessageOneLine(logMessage)}`)
+      .map(
+        (logMessage) =>
+          `${format(
+            parseISO(logMessage.ts),
+            'HH:mm:ss'
+          )} - ${formatLogMessageOneLine(logMessage)}`
+      )
       .map((logMessage) => console.log(logMessage))
 
     return
   }
   console.log('No errors or warnings occurred')
 }
 
-// Write all log messages instead of infos to the error log file
-export function writeErrorLogFile (destination, errorLog) {
-  const logFileData = errorLog
-    .map(formatLogMessageLogfile)
-
-  return bfj.write(destination, logFileData, {
-    circular: 'ignore',
-    space: 2
+/**
+ * Write all log messages instead of infos to the error log file
+ * @param {import('node:fs').PathLike} destination
+ * @param {Record<string, unknown>[]} errorLog
+ * @returns {Promise<void>}
+ */
+export async function writeErrorLogFile (destination, errorLog) {
+  const formatLogTransformer = new Transform({
+    objectMode: true,
+    transform: (chunk, encoding, callback) => {
+      const formattedChunk = formatLogMessageLogfile(chunk)
+      callback(null, Buffer.from(JSON.stringify(formattedChunk)))
+    }
   })
-    .then(() => {
-      console.log('\nStored the detailed error log file at:')
-      console.log(destination)
-    })
-    .catch((e) => {
-      // avoid crashing when writing the log file fails
-      console.error(e)
-    })
+
+  const logFileWriteStream = createWriteStream(destination)
+
+  try {
+    await pipeline([
+      Readable.from(errorLog),
+      formatLogTransformer,
+      logFileWriteStream
+    ])
+
+    console.log('\nStored the detailed error log file at:')
+    console.log(destination)
+  } catch (err) {
+    // avoid crashing when writing the log file fails
+    console.error(err)
+  }
 }
 
-// Init listeners for log messages, transform them into proper format and logs/displays them
+/**
+ * Init listeners for log messages, transform them into proper format and logs/displays them
+ * @param {Record<string,unknown>[]} log
+ * @returns {Promise<void>}
+ */
 export function setupLogging (log) {
   function errorLogger (level, error) {
     const logMessage = {
-      ts: (new Date()).toJSON(),
+      ts: new Date().toJSON(),
       level,
       [level]: error
     }
@@ -149,7 +184,11 @@ export function setupLogging (log) {
   logEmitter.addListener('error', (error) => errorLogger('error', error))
 }
 
-// Format log message to display them as task status
+/**
+ * Format log message to display them as task status
+ * @template {Ctx}
+ * @param {import('listr').ListrTaskWrapper<Ctx>} task
+ */
 export function logToTaskOutput (task) {
   function logToTask (logMessage) {
     const content = formatLogMessageOneLine(logMessage)