From 95bd299eb6043f7cbef8dc32a38a479b75b5b986 Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 13:11:39 +0100 Subject: [PATCH 1/8] =?UTF-8?q?Setter=20rate=20limit=20til=20100=20request?= =?UTF-8?q?s=20p=C3=A5=201s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/package-lock.json | 21 +++++++++++++++++++++ server/package.json | 1 + server/server.js | 12 ++++++++++++ 3 files changed, 34 insertions(+) diff --git a/server/package-lock.json b/server/package-lock.json index 48ace3b4d..7d12f69e4 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -12,6 +12,7 @@ "@navikt/oasis": "3.4.0", "express": "^4.21.2", "express-http-proxy": "2.0.0", + "express-rate-limit": "7.5.0", "http-proxy-middleware": "3.0.3", "http-terminator": "3.2.0", "jsdom": "^24.1.0", @@ -709,6 +710,20 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/express-rate-limit": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.0.tgz", + "integrity": "sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg==", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": "^4.11 || 5 || ^5.0.0-beta.1" + } + }, "node_modules/express/node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -2700,6 +2715,12 @@ } } }, + "express-rate-limit": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.0.tgz", + "integrity": "sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg==", + "requires": {} + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", diff --git a/server/package.json b/server/package.json index 832814cbc..06ac20143 100644 --- a/server/package.json +++ b/server/package.json @@ -8,6 +8,7 @@ "@navikt/oasis": "3.4.0", "express": "^4.21.2", "express-http-proxy": "2.0.0", + "express-rate-limit": "7.5.0", "http-proxy-middleware": "3.0.3", "http-terminator": "3.2.0", "jsdom": "^24.1.0", diff --git a/server/server.js b/server/server.js index b1820bc87..5798b0bc9 100644 --- a/server/server.js +++ b/server/server.js @@ -13,6 +13,7 @@ import { createLogger, format, transports } from 'winston'; import { tokenXMiddleware } from './tokenx.js'; import { readFileSync } from 'fs'; import require from './esm-require.js'; +import { rateLimit } from 'express-rate-limit' const apiMetricsMiddleware = require('prometheus-api-metrics'); const { createProxyMiddleware } = httpProxyMiddleware; @@ -42,6 +43,14 @@ const maskFormat = format((info) => ({ message: info.message.replace(/\d{9,}/g, (match) => '*'.repeat(match.length)), })); +const apiRateLimit = rateLimit({ + windowMs: 1000, // 1 sekund + limit: 100, + message: 'You have exceeded the 100 requests in 1s limit!', + standardHeaders: true, + legacyHeaders: false, +}) + // proxy calls to log. https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy/Proxy/get const log = new Proxy( createLogger({ @@ -172,6 +181,9 @@ const main = async () => { let appReady = false; const app = express(); app.disable('x-powered-by'); + + app.use(apiRateLimit) + app.set('views', BUILD_PATH); app.use('/*', (req, res, next) => { From 8a4e7c5645bbbd6d1f6e51c6850452f34949018d Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 13:47:03 +0100 Subject: [PATCH 2/8] =?UTF-8?q?Setter=20rate=20limit=20til=20200=20request?= =?UTF-8?q?s=20p=C3=A5=201m?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/server.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/server.js b/server/server.js index 5798b0bc9..9185f5109 100644 --- a/server/server.js +++ b/server/server.js @@ -44,9 +44,9 @@ const maskFormat = format((info) => ({ })); const apiRateLimit = rateLimit({ - windowMs: 1000, // 1 sekund - limit: 100, - message: 'You have exceeded the 100 requests in 1s limit!', + windowMs: 60 * 1000, // 1 minutt + limit: 200, + message: 'You have exceeded the 200 requests in 1m limit!', standardHeaders: true, legacyHeaders: false, }) From 4a486208b4872d43576feb1f2f220b66a9d3ea69 Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 13:56:16 +0100 Subject: [PATCH 3/8] =?UTF-8?q?Setter=20rate=20limit=20til=20100=20request?= =?UTF-8?q?s=20p=C3=A5=201s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/server.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/server.js b/server/server.js index 9185f5109..5798b0bc9 100644 --- a/server/server.js +++ b/server/server.js @@ -44,9 +44,9 @@ const maskFormat = format((info) => ({ })); const apiRateLimit = rateLimit({ - windowMs: 60 * 1000, // 1 minutt - limit: 200, - message: 'You have exceeded the 200 requests in 1m limit!', + windowMs: 1000, // 1 sekund + limit: 100, + message: 'You have exceeded the 100 requests in 1s limit!', standardHeaders: true, legacyHeaders: false, }) From 6d7cc2b19075a4ca859fa67b904dcaff6b8386c1 Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 14:35:25 +0100 Subject: [PATCH 4/8] =?UTF-8?q?Legger=20til=20logging=20hvis=20rate=20limi?= =?UTF-8?q?t=20er=20n=C3=A5dd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/server.js | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/server/server.js b/server/server.js index 5798b0bc9..96948ebb0 100644 --- a/server/server.js +++ b/server/server.js @@ -43,14 +43,6 @@ const maskFormat = format((info) => ({ message: info.message.replace(/\d{9,}/g, (match) => '*'.repeat(match.length)), })); -const apiRateLimit = rateLimit({ - windowMs: 1000, // 1 sekund - limit: 100, - message: 'You have exceeded the 100 requests in 1s limit!', - standardHeaders: true, - legacyHeaders: false, -}) - // proxy calls to log. https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy/Proxy/get const log = new Proxy( createLogger({ @@ -72,6 +64,21 @@ const log = new Proxy( } ); + +const apiRateLimit = rateLimit({ + windowMs: 1000, // 1 sekund + limit: 100, // Limit each IP to 100 requests per `window` + message: 'You have exceeded the 100 requests in 1s limit!', + standardHeaders: true, + legacyHeaders: false, + handler: (req, res, next, options) => { + if (req.rateLimit.remaning === 0) { + log('error', `Rate limit reached for IP: ${req.ip}`); + } + res.status(options.statusCode).send(options.message); + } +}) + const cookieScraperPlugin = (proxyServer, options) => { proxyServer.on('proxyReq', (proxyReq, req, res, options) => { if (proxyReq.getHeader('cookie')) { From e5b36f0985c6e9499e077416946c7de64cda283e Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 14:37:01 +0100 Subject: [PATCH 5/8] =?UTF-8?q?Legger=20til=20logging=20hvis=20rate=20limi?= =?UTF-8?q?t=20er=20n=C3=A5dd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server.js b/server/server.js index 96948ebb0..3af02fbbb 100644 --- a/server/server.js +++ b/server/server.js @@ -72,8 +72,8 @@ const apiRateLimit = rateLimit({ standardHeaders: true, legacyHeaders: false, handler: (req, res, next, options) => { - if (req.rateLimit.remaning === 0) { - log('error', `Rate limit reached for IP: ${req.ip}`); + if (req.rateLimit.remaining === 0) { + log.error(`Rate limit reached for IP: ${req.ip}`); } res.status(options.statusCode).send(options.message); } From 261ab7b78e9380cf564e8cc8848be5287e1ab72c Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 16:23:49 +0100 Subject: [PATCH 6/8] Setter rate limit per auth token med ip som fallback --- server/server.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/server.js b/server/server.js index 3af02fbbb..4b526eca0 100644 --- a/server/server.js +++ b/server/server.js @@ -64,20 +64,20 @@ const log = new Proxy( } ); - const apiRateLimit = rateLimit({ windowMs: 1000, // 1 sekund limit: 100, // Limit each IP to 100 requests per `window` message: 'You have exceeded the 100 requests in 1s limit!', standardHeaders: true, legacyHeaders: false, + keyGenerator: (req) => req.headers?.authorization?.replace('Bearer', '') || req.ip, handler: (req, res, next, options) => { - if (req.rateLimit.remaining === 0) { - log.error(`Rate limit reached for IP: ${req.ip}`); - } - res.status(options.statusCode).send(options.message); + if (req.rateLimit.remaining === 0) { + log.error(`Rate limit reached for client ${req.ip}`); + } + res.status(options.statusCode).send(options.message); } -}) +}); const cookieScraperPlugin = (proxyServer, options) => { proxyServer.on('proxyReq', (proxyReq, req, res, options) => { From 567826a083c223dc13019351b5269fb487596c86 Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 16:24:23 +0100 Subject: [PATCH 7/8] Setter rate limit per auth token med ip som fallback --- server/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.js b/server/server.js index 4b526eca0..9ac5354bf 100644 --- a/server/server.js +++ b/server/server.js @@ -70,7 +70,7 @@ const apiRateLimit = rateLimit({ message: 'You have exceeded the 100 requests in 1s limit!', standardHeaders: true, legacyHeaders: false, - keyGenerator: (req) => req.headers?.authorization?.replace('Bearer', '') || req.ip, + keyGenerator: (req) => req.headers?.authorization?.replace('Bearer ', '') || req.ip, handler: (req, res, next, options) => { if (req.rateLimit.remaining === 0) { log.error(`Rate limit reached for client ${req.ip}`); From 3343b68ed04e5e9e1df2733d8aed3b15be1eafbb Mon Sep 17 00:00:00 2001 From: Jonas Enge Date: Tue, 18 Feb 2025 16:54:00 +0100 Subject: [PATCH 8/8] Hasher tokenet for mindre memory footprint i cache --- server/server.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/server.js b/server/server.js index 9ac5354bf..e2e6adb4e 100644 --- a/server/server.js +++ b/server/server.js @@ -14,6 +14,7 @@ import { tokenXMiddleware } from './tokenx.js'; import { readFileSync } from 'fs'; import require from './esm-require.js'; import { rateLimit } from 'express-rate-limit' +import crypto from 'crypto' const apiMetricsMiddleware = require('prometheus-api-metrics'); const { createProxyMiddleware } = httpProxyMiddleware; @@ -64,13 +65,22 @@ const log = new Proxy( } ); +const hashToken = token => crypto.createHash('sha256').update(token).digest('base64'); + const apiRateLimit = rateLimit({ windowMs: 1000, // 1 sekund limit: 100, // Limit each IP to 100 requests per `window` message: 'You have exceeded the 100 requests in 1s limit!', standardHeaders: true, legacyHeaders: false, - keyGenerator: (req) => req.headers?.authorization?.replace('Bearer ', '') || req.ip, + keyGenerator: (req) => { + const authHeader = req.headers?.authorization || ''; + if (!authHeader.startsWith('Bearer ')) { + return req.ip; + } + const token = authHeader.substring(7); + return hashToken(token); + }, handler: (req, res, next, options) => { if (req.rateLimit.remaining === 0) { log.error(`Rate limit reached for client ${req.ip}`);