test: add end-to-end tests for opentelemetry

I wasn't super confident after the recent major update and it led to me
doing lots of tests across apps, symlinking in the new package version,
and manually checking that metrics and traces were still being sent.

I realised that this could be automated. I've added an end-to-end test
where we spin up two Node.js apps - one collector and one regular app
with OpenTelemetry added. We capture logs from both and this allows us
to verify the basic case of "do metrics/traces get from an app to the
configured collector?"

Author: rowanmanning

package-lock.json

@@ -28,5 +28,8 @@
 		"@opentelemetry/instrumentation-runtime-node": "^0.13.0",
 		"@opentelemetry/sdk-node": "^0.200.0",
 		"@opentelemetry/semantic-conventions": "^1.30.0"
+	},
+	"devDependencies": {
+		"express": "^4.21.2"
 	}
 }

packages/opentelemetry/package.json

@@ -0,0 +1,34 @@
+const express = require('express');
+const logger = require('@dotcom-reliability-kit/logger');
+
+// We set up OpenTelemetry via `--require` but this allows us to grab the same instances
+const { setup } = require('@dotcom-reliability-kit/opentelemetry');
+const { sdk } = setup();
+
+const app = express();
+
+app.use(async (request, response) => {
+	logger.info({
+		event: 'INCOMING_REQUEST',
+		method: request.method,
+		url: request.url
+	});
+
+	// This ensures that metrics and traces are flushed
+	await sdk.shutdown();
+
+	response.status(200).send('');
+});
+
+const server = app.listen((error) => {
+	if (error) {
+		logger.fatal(`App could not be started: ${error.message}`);
+		return process.exit(1);
+	}
+	if (process.send) {
+		process.send({
+			ready: true,
+			port: server.address().port
+		});
+	}
+});

packages/opentelemetry/test/end-to-end/fixtures/app.js

@@ -0,0 +1,35 @@
+const express = require('express');
+const logger = require('@dotcom-reliability-kit/logger');
+
+const app = express();
+
+app.use(express.raw({ type: 'application/x-protobuf' }));
+
+app.use(async (request, response) => {
+	try {
+		logger.info({
+			event: 'INCOMING_REQUEST',
+			method: request.method,
+			url: request.url,
+			headers: request.headers,
+			body: request.body.toString()
+		});
+		response.status(200).send('');
+	} catch (error) {
+		logger.error(error);
+		response.status(500).send('');
+	}
+});
+
+const server = app.listen((error) => {
+	if (error) {
+		logger.fatal(`Collector could not be started: ${error.message}`);
+		return process.exit(1);
+	}
+	if (process.send) {
+		process.send({
+			ready: true,
+			port: server.address().port
+		});
+	}
+});

packages/opentelemetry/test/end-to-end/fixtures/collector.js

@@ -0,0 +1,140 @@
+const { fork } = require('node:child_process');
+
+function waitForBaseUrl(childProcess) {
+	return new Promise((resolve) => {
+		const handler = (message) => {
+			if (message?.ready) {
+				resolve(`http://localhost:${message.port}`);
+				childProcess.off('message', handler);
+			}
+		};
+		childProcess.on('message', handler);
+	});
+}
+
+function stdoutToLogs(stdout) {
+	return stdout.split('\n').map((logLine) => {
+		try {
+			return JSON.parse(logLine);
+		} catch (_) {
+			return logLine;
+		}
+	});
+}
+
+describe('@dotcom-reliability-kit/opentelemetry end-to-end', () => {
+	let collector;
+	let collectorStdout = '';
+	let collectorBaseUrl;
+	let exporter;
+	let exporterStdout = '';
+	let exporterBaseUrl;
+
+	beforeAll(async () => {
+		// Set up a mock collector
+		collector = fork(`${__dirname}/fixtures/collector.js`, {
+			env: {
+				...process.env,
+				NODE_ENV: 'production',
+				SYSTEM_CODE: 'mock-system'
+			},
+			stdio: 'pipe'
+		});
+		collector.stdout.on('data', (chunk) => {
+			collectorStdout += chunk.toString();
+		});
+		collector.stderr.on('data', (chunk) => {
+			collectorStdout += chunk.toString();
+		});
+		collectorBaseUrl = await waitForBaseUrl(collector);
+
+		// Set up a Node.js app that sends Opentelemetry metrics and traces
+		exporter = fork(`${__dirname}/fixtures/app.js`, {
+			env: {
+				...process.env,
+				NODE_ENV: 'production',
+				OPENTELEMETRY_METRICS_ENDPOINT: `${collectorBaseUrl}/metrics`,
+				OPENTELEMETRY_TRACING_ENDPOINT: `${collectorBaseUrl}/traces`,
+				OPENTELEMETRY_TRACING_SAMPLE_PERCENTAGE: '100',
+				SYSTEM_CODE: 'mock-system'
+			},
+			execArgv: ['--require', '@dotcom-reliability-kit/opentelemetry/setup'],
+			stdio: 'pipe'
+		});
+		exporter.stdout.on('data', (chunk) => {
+			exporterStdout += chunk.toString();
+		});
+		exporter.stderr.on('data', (chunk) => {
+			exporterStdout += chunk.toString();
+		});
+		exporterBaseUrl = await waitForBaseUrl(exporter);
+	});
+
+	afterAll(() => {
+		if (collector) {
+			collector.kill('SIGINT');
+		}
+		if (exporter) {
+			exporter.kill('SIGINT');
+		}
+	});
+
+	describe('sending an HTTP request to the exporting app', () => {
+		let collectorLogs;
+		let exporterLogs;
+
+		beforeAll(async () => {
+			try {
+				await fetch(`${exporterBaseUrl}/example`);
+				collectorLogs = stdoutToLogs(collectorStdout);
+				exporterLogs = stdoutToLogs(exporterStdout);
+			} catch (cause) {
+				// eslint-disable-next-line no-console
+				console.log('COLLECTOR:', stdoutToLogs(collectorStdout));
+				// eslint-disable-next-line no-console
+				console.log('EXPORTER:', stdoutToLogs(exporterStdout));
+				throw new Error('Fetch failed, see logs');
+			}
+		});
+
+		describe('exporter', () => {
+			it('logs that OpenTelemetry metrics are enabled', () => {
+				const log = exporterLogs.find(
+					(log) => log?.event === 'OTEL_METRICS_STATUS' && log?.enabled === true
+				);
+				expect(log).toBeDefined();
+			});
+			it('logs that OpenTelemetry tracing is enabled', () => {
+				const log = exporterLogs.find(
+					(log) => log?.event === 'OTEL_TRACE_STATUS' && log?.enabled === true
+				);
+				expect(log).toBeDefined();
+			});
+		});
+
+		describe('collector', () => {
+			it('receives metrics', () => {
+				const log = collectorLogs.find(
+					(log) =>
+						log?.event === 'INCOMING_REQUEST' &&
+						log?.method === 'POST' &&
+						log?.url === '/metrics'
+				);
+				expect(log).toBeDefined();
+				expect(log.headers['content-type']).toBe('application/x-protobuf');
+				expect(log.body).toContain('mock-system');
+			});
+			it('receives traces', () => {
+				const log = collectorLogs.find(
+					(log) =>
+						log?.event === 'INCOMING_REQUEST' &&
+						log?.method === 'POST' &&
+						log?.url === '/traces'
+				);
+				expect(log).toBeDefined();
+				expect(log.headers['content-type']).toBe('application/x-protobuf');
+				expect(log.body).toContain('mock-system');
+			});
+		});
+	});
+});