Merge pull request #232 from AikidoSec/rate-limit-ip

Rate limit by IP + build route from URL

Author: willem-delbare

library/agent/Agent.ts

@@ -421,6 +421,10 @@ export class Agent {
     this.routes.addRoute(method, path);
   }
 
+  getRoutes() {
+    return this.routes;
+  }
+
   async flushStats(timeoutInMS: number) {
     this.statistics.forceCompress();
     await this.sendHeartbeat(timeoutInMS);

library/helpers/buildRouteFromURL.test.ts

@@ -0,0 +1,111 @@
+import * as t from "tap";
+import { buildRouteFromURL } from "./buildRouteFromURL";
+
+t.test("it returns undefined for invalid URLs", async () => {
+  t.same(buildRouteFromURL(""), undefined);
+  t.same(buildRouteFromURL("http"), undefined);
+});
+
+t.test("it returns / for root URLs", async () => {
+  t.same(buildRouteFromURL("/"), "/");
+  t.same(buildRouteFromURL("http://localhost/"), "/");
+});
+
+t.test("it replaces numbers", async () => {
+  t.same(buildRouteFromURL("/posts/3"), "/posts/:number");
+  t.same(buildRouteFromURL("http://localhost/posts/3"), "/posts/:number");
+  t.same(buildRouteFromURL("http://localhost/posts/3/"), "/posts/:number");
+  t.same(
+    buildRouteFromURL("http://localhost/posts/3/comments/10"),
+    "/posts/:number/comments/:number"
+  );
+  t.same(
+    buildRouteFromURL("/blog/2023/05/great-article"),
+    "/blog/:number/:number/great-article"
+  );
+});
+
+t.test("it replaces dates", async () => {
+  t.same(buildRouteFromURL("/posts/2023-05-01"), "/posts/:date");
+  t.same(buildRouteFromURL("/posts/2023-05-01/"), "/posts/:date");
+  t.same(
+    buildRouteFromURL("/posts/2023-05-01/comments/2023-05-01"),
+    "/posts/:date/comments/:date"
+  );
+  t.same(buildRouteFromURL("/posts/01-05-2023"), "/posts/:date");
+});
+
+t.test("it ignores comma numbers", async () => {
+  t.same(buildRouteFromURL("/posts/3,000"), "/posts/3,000");
+});
+
+t.test("it ignores API version numbers", async () => {
+  t.same(buildRouteFromURL("/v1/posts/3"), "/v1/posts/:number");
+});
+
+t.test("it replaces UUIDs v1", async () => {
+  t.same(
+    buildRouteFromURL("/posts/d9428888-122b-11e1-b85c-61cd3cbb3210"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it replaces UUIDs v2", async () => {
+  t.same(
+    buildRouteFromURL("/posts/000003e8-2363-21ef-b200-325096b39f47"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it replaces UUIDs v3", async () => {
+  t.same(
+    buildRouteFromURL("/posts/a981a0c2-68b1-35dc-bcfc-296e52ab01ec"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it replaces UUIDs v4", async () => {
+  t.same(
+    buildRouteFromURL("/posts/109156be-c4fb-41ea-b1b4-efe1671c5836"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it replaces UUIDs v5", async () => {
+  t.same(
+    buildRouteFromURL("/posts/90123e1c-7512-523e-bb28-76fab9f2f73d"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it replaces UUIDs v6", async () => {
+  t.same(
+    buildRouteFromURL("/posts/1ef21d2f-1207-6660-8c4f-419efbd44d48"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it replaces UUIDs v7", async () => {
+  t.same(
+    buildRouteFromURL("/posts/017f22e2-79b0-7cc3-98c4-dc0c0c07398f"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it replaces UUIDs v8", async () => {
+  t.same(
+    buildRouteFromURL("/posts/0d8f23a0-697f-83ae-802e-48f3756dd581"),
+    "/posts/:uuid"
+  );
+});
+
+t.test("it ignores invalid UUIDs", async () => {
+  t.same(
+    buildRouteFromURL("/posts/00000000-0000-1000-6000-000000000000"),
+    "/posts/00000000-0000-1000-6000-000000000000"
+  );
+});
+
+t.test("it ignores strings", async () => {
+  t.same(buildRouteFromURL("/posts/abc"), "/posts/abc");
+});

library/helpers/buildRouteFromURL.ts

@@ -0,0 +1,42 @@
+import { tryParseURLPath } from "./tryParseURLPath";
+
+const UUID =
+  /(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$/i;
+const NUMBER = /^\d+$/;
+const DATE = /^\d{4}-\d{2}-\d{2}|\d{2}-\d{2}-\d{4}$/;
+
+export function buildRouteFromURL(url: string) {
+  const path = tryParseURLPath(url);
+
+  if (!path) {
+    return undefined;
+  }
+
+  const route = path.split("/").map(replaceURLSegmentWithParam).join("/");
+
+  if (route === "/") {
+    return "/";
+  }
+
+  if (route.endsWith("/")) {
+    return route.slice(0, -1);
+  }
+
+  return route;
+}
+
+function replaceURLSegmentWithParam(segment: string) {
+  if (NUMBER.test(segment)) {
+    return ":number";
+  }
+
+  if (UUID.test(segment)) {
+    return ":uuid";
+  }
+
+  if (DATE.test(segment)) {
+    return ":date";
+  }
+
+  return segment;
+}

library/helpers/matchEndpoint.test.ts

@@ -1,18 +1,20 @@
 import * as t from "tap";
 import { Context } from "../agent/Context";
+import { buildRouteFromURL } from "./buildRouteFromURL";
 import { matchEndpoint } from "./matchEndpoint";
 
+const url = "http://localhost:4000/posts/3";
 const context: Context = {
   remoteAddress: "::1",
   method: "POST",
-  url: "http://localhost:4000/posts/3",
+  url: url,
   query: {},
   headers: {},
   body: undefined,
   cookies: {},
   routeParams: {},
   source: "express",
-  route: "/posts/:id",
+  route: buildRouteFromURL(url),
 };
 
 t.test("invalid URL and no route", async () => {
@@ -42,48 +44,77 @@ t.test("it returns endpoint based on route", async () => {
     matchEndpoint(context, [
       {
         method: "POST",
-        route: "/posts/:id",
+        route: "/posts/:number",
         rateLimiting: { enabled: true, maxRequests: 10, windowSizeInMS: 1000 },
         forceProtectionOff: false,
       },
     ]),
     {
       endpoint: {
         method: "POST",
-        route: "/posts/:id",
+        route: "/posts/:number",
         rateLimiting: { enabled: true, maxRequests: 10, windowSizeInMS: 1000 },
         forceProtectionOff: false,
       },
-      route: "/posts/:id",
+      route: "/posts/:number",
     }
   );
 });
 
-t.test("it returns endpoint based on url", async () => {
+t.test("it returns endpoint based on relative url", async () => {
+  t.same(
+    matchEndpoint(
+      { ...context, route: buildRouteFromURL("/posts/3"), url: "/posts/3" },
+      [
+        {
+          method: "POST",
+          route: "/posts/:number",
+          rateLimiting: {
+            enabled: true,
+            maxRequests: 10,
+            windowSizeInMS: 1000,
+          },
+          forceProtectionOff: false,
+        },
+      ]
+    ),
+    {
+      endpoint: {
+        method: "POST",
+        route: "/posts/:number",
+        rateLimiting: { enabled: true, maxRequests: 10, windowSizeInMS: 1000 },
+        forceProtectionOff: false,
+      },
+      route: "/posts/:number",
+    }
+  );
+});
+
+t.test("it returns endpoint based on wildcard", async () => {
   t.same(
     matchEndpoint({ ...context, route: undefined }, [
       {
-        method: "POST",
-        route: "/posts/3",
+        method: "*",
+        route: "/posts/*",
         rateLimiting: { enabled: true, maxRequests: 10, windowSizeInMS: 1000 },
         forceProtectionOff: false,
       },
     ]),
     {
       endpoint: {
-        method: "POST",
-        route: "/posts/3",
+        method: "*",
+        route: "/posts/*",
         rateLimiting: { enabled: true, maxRequests: 10, windowSizeInMS: 1000 },
         forceProtectionOff: false,
       },
-      route: "/posts/3",
+      route: "/posts/*",
     }
   );
 });
 
-t.test("it returns endpoint based on wildcard", async () => {
+t.test("it returns endpoint based on wildcard with relative URL", async () => {
   t.same(
-    matchEndpoint({ ...context, route: undefined }, [
+    matchEndpoint({ ...context, route: undefined, url: "/posts/3" }, [
       {
         method: "*",
         route: "/posts/*",
@@ -146,50 +177,6 @@ t.test("it favors more specific wildcard", async () => {
   );
 });
 
-t.test("it does not check whether rate limiting is enabled", async () => {
-  t.same(
-    matchEndpoint({ ...context, route: undefined }, [
-      {
-        method: "POST",
-        route: "/posts/3",
-        rateLimiting: { enabled: false, maxRequests: 10, windowSizeInMS: 1000 },
-        forceProtectionOff: false,
-      },
-    ]),
-    {
-      endpoint: {
-        method: "POST",
-        route: "/posts/3",
-        rateLimiting: { enabled: false, maxRequests: 10, windowSizeInMS: 1000 },
-        forceProtectionOff: false,
-      },
-      route: "/posts/3",
-    }
-  );
-});
-
-t.test("it does not check whether force protection is off", async () => {
-  t.same(
-    matchEndpoint({ ...context, route: undefined }, [
-      {
-        method: "POST",
-        route: "/posts/3",
-        rateLimiting: { enabled: true, maxRequests: 10, windowSizeInMS: 1000 },
-        forceProtectionOff: true,
-      },
-    ]),
-    {
-      endpoint: {
-        method: "POST",
-        route: "/posts/3",
-        rateLimiting: { enabled: true, maxRequests: 10, windowSizeInMS: 1000 },
-        forceProtectionOff: true,
-      },
-      route: "/posts/3",
-    }
-  );
-});
-
 t.test("it matches wildcard route with specific method", async () => {
   t.same(
     matchEndpoint(
@@ -229,7 +216,7 @@ t.test("it prefers specific route over wildcard", async () => {
     matchEndpoint(
       {
         ...context,
-        route: undefined,
+        route: "/api/coach",
         method: "POST",
         url: "http://localhost:4000/api/coach",
       },

library/helpers/matchEndpoint.ts

@@ -1,6 +1,6 @@
 import { Endpoint } from "../agent/Config";
 import { Context } from "../agent/Context";
-import { tryParseURL } from "./tryParseURL";
+import { tryParseURLPath } from "./tryParseURLPath";
 
 export type LimitedContext = Pick<Context, "url" | "method" | "route">;
 
@@ -17,32 +17,26 @@ export function matchEndpoint(context: LimitedContext, endpoints: Endpoint[]) {
     return endpoint.method === context.method;
   });
 
-  if (context.route) {
-    const endpoint = possible.find(
-      (endpoint) => endpoint.route === context.route
-    );
+  const endpoint = possible.find(
+    (endpoint) => endpoint.route === context.route
+  );
 
-    if (endpoint) {
-      return { endpoint: endpoint, route: context.route };
-    }
+  if (endpoint) {
+    return { endpoint: endpoint, route: endpoint.route };
   }
 
   if (!context.url) {
     return undefined;
   }
 
-  const url = tryParseURL(context.url);
+  // req.url is relative, so we need to prepend a host to make it absolute
+  // We just match the pathname, we don't use the host for matching
+  const path = tryParseURLPath(context.url);
 
-  if (!url || !url.pathname) {
+  if (!path) {
     return undefined;
   }
 
-  const endpoint = possible.find((endpoint) => endpoint.route === url.pathname);
-
-  if (endpoint) {
-    return { endpoint: endpoint, route: endpoint.route };
-  }
-
   const wildcards = possible
     .filter((endpoint) => endpoint.route.includes("*"))
     .sort((a, b) => {
@@ -56,7 +50,7 @@ export function matchEndpoint(context: LimitedContext, endpoints: Endpoint[]) {
       "i"
     );
 
-    if (regex.test(url.pathname)) {
+    if (regex.test(path)) {
       return { endpoint: wildcard, route: wildcard.route };
     }
   }