thirdweb-dev · arcoraven · Feb 9, 2025
diff --git a/package.json b/package.json
@@ -43,7 +43,7 @@
     "@thirdweb-dev/auth": "^4.1.87",
     "@thirdweb-dev/chains": "^0.1.77",
     "@thirdweb-dev/sdk": "^4.0.89",
-    "@thirdweb-dev/service-utils": "^0.4.28",
+    "@thirdweb-dev/service-utils": "^0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117",
     "@types/base-64": "^1.0.2",
     "aws-kms-signer": "^0.5.3",
     "base-64": "^1.0.0",

diff --git a/src/shared/utils/env.ts b/src/shared/utils/env.ts
@@ -54,6 +54,7 @@ export const env = createEnv({
     CLIENT_ANALYTICS_URL: z
       .union([UrlSchema, z.literal("")])
       .default("https://c.thirdweb.com/event"),
+    ENABLE_USAGE_V2_ANALYTICS: boolEnvSchema(true),
     SDK_BATCH_TIME_LIMIT: z.coerce.number().default(0),
     SDK_BATCH_SIZE_LIMIT: z.coerce.number().default(100),
     REDIS_URL: z.string(),
@@ -120,6 +121,7 @@ export const env = createEnv({
     HTTPS_PASSPHRASE: process.env.HTTPS_PASSPHRASE,
     TRUST_PROXY: process.env.TRUST_PROXY,
     CLIENT_ANALYTICS_URL: process.env.CLIENT_ANALYTICS_URL,
+    ENABLE_USAGE_V2_ANALYTICS: process.env.ENABLE_USAGE_V2_ANALYTICS,
     SDK_BATCH_TIME_LIMIT: process.env.SDK_BATCH_TIME_LIMIT,
     SDK_BATCH_SIZE_LIMIT: process.env.SDK_BATCH_SIZE_LIMIT,
     REDIS_URL: process.env.REDIS_URL,

diff --git a/src/shared/utils/usage.ts b/src/shared/utils/usage.ts
@@ -1,5 +1,9 @@
 import type { Static } from "@sinclair/typebox";
-import type { UsageEvent } from "@thirdweb-dev/service-utils/cf-worker";
+import {
+  sendUsageV2Events,
+  type ClientUsageV2Event,
+  type UsageEvent,
+} from "@thirdweb-dev/service-utils/cf-worker";
 import type { FastifyInstance } from "fastify";
 import type { Address, Hex } from "thirdweb";
 import { ADMIN_QUEUES_BASEPATH } from "../../server/middleware/admin-routes";
@@ -11,7 +15,7 @@ import { env } from "./env";
 import { logger } from "./logger";
 import { thirdwebClientId } from "./sdk";
 
-export interface ReportUsageParams {
+interface ReportTransactionUsageParams {
   action:
     | "queue_tx"
     | "not_send_tx"
@@ -26,7 +30,7 @@ export interface ReportUsageParams {
     to?: Address;
     value?: bigint;
     transactionHash?: Hex;
-    onChainTxStatus?: number;
+    onchainStatus?: "success" | "reverted";
     userOpHash?: Hex;
     functionName?: string;
     extension?: string;
@@ -38,13 +42,6 @@ export interface ReportUsageParams {
   error?: string;
 }
 
-const ANALYTICS_DEFAULT_HEADERS = {
-  "Content-Type": "application/json",
-  "x-sdk-version": process.env.ENGINE_VERSION,
-  "x-product-name": "engine",
-  "x-client-id": thirdwebClientId,
-} as HeadersInit;
-
 const SKIP_USAGE_PATHS = new Set([
   "",
   "/",
@@ -55,8 +52,8 @@ const SKIP_USAGE_PATHS = new Set([
 ]);
 
 export function withServerUsageReporting(server: FastifyInstance) {
-  // Skip reporting if CLIENT_ANALYTICS_URL is unset.
-  if (env.CLIENT_ANALYTICS_URL === "") {
+  // Skip reporting if analytics reporting is disabled.
+  if (!env.CLIENT_ANALYTICS_URL && !env.ENABLE_USAGE_V2_ANALYTICS) {
     return;
   }
 
@@ -79,66 +76,113 @@ export function withServerUsageReporting(server: FastifyInstance) {
       ? await getChainIdFromChain(requestParams.chain)
       : undefined;
 
-    const requestBody: UsageEvent = {
-      source: "engine",
-      action: "api_request",
-      clientId: thirdwebClientId,
-      pathname: reply.request.routeOptions.url,
-      chainId,
-      walletAddress: requestParams?.walletAddress,
-      contractAddress: requestParams?.contractAddress,
-      httpStatusCode: reply.statusCode,
-      msTotalDuration: Math.ceil(reply.elapsedTime),
-      httpMethod: request.method.toUpperCase() as UsageEvent["httpMethod"],
-    };
-
-    fetch(env.CLIENT_ANALYTICS_URL, {
-      method: "POST",
-      headers: ANALYTICS_DEFAULT_HEADERS,
-      body: JSON.stringify(requestBody),
-    }).catch(() => {}); // Catch uncaught exceptions since this fetch call is non-blocking.
+    await reportUsageV1([
+      {
+        source: "engine",
+        action: "api_request",
+        clientId: thirdwebClientId,
+        pathname: reply.request.routeOptions.url,
+        chainId,
+        walletAddress: requestParams?.walletAddress,
+        contractAddress: requestParams?.contractAddress,
+        httpStatusCode: reply.statusCode,
+        msTotalDuration: Math.ceil(reply.elapsedTime),
+        httpMethod: request.method.toUpperCase() as UsageEvent["httpMethod"],
+      },
+    ]);
+    // @TODO: UsageV2 reporting
   });
 }
 
-export const reportUsage = (usageEvents: ReportUsageParams[]) => {
-  // Skip reporting if CLIENT_ANALYTICS_URL is unset.
-  if (env.CLIENT_ANALYTICS_URL === "") {
-    return;
+/**
+ * Reports usage events.
+ * This method must not throw uncaught exceptions and is safe to call non-blocking.
+ */
+export async function reportUsage(events: ReportTransactionUsageParams[]) {
+  try {
+    if (env.CLIENT_ANALYTICS_URL) {
+      await reportUsageV1(
+        events.map(({ action, input, error }) => ({
+          source: "engine",
+          action,
+          clientId: thirdwebClientId,
+          chainId: input.chainId,
+          walletAddress: input.from,
+          contractAddress: input.to,
+          transactionValue: input.value?.toString(),
+          transactionHash: input.transactionHash,
+          userOpHash: input.userOpHash,
+          errorCode:
+            input.onchainStatus === "reverted" ? "EXECUTION_REVERTED" : error,
+          functionName: input.functionName,
+          extension: input.extension,
+          retryCount: input.retryCount,
+          provider: input.provider,
+          msSinceQueue: input.msSinceQueue,
+          msSinceSend: input.msSinceSend,
+        })),
+      );
+    }
+    if (env.ENABLE_USAGE_V2_ANALYTICS) {
+      await reportUsageV2(
+        events.map((event) => ({
+          action: event.action,
+          sdk_platform: env.ENGINE_TIER ?? "SELF_HOSTED",
+          sdk_version: env.ENGINE_VERSION,
+          product_name: "engine",
+          chain_id: event.input.chainId,
+          from_address: event.input.from,
+          to_address: event.input.to,
+          value: event.input.value?.toString(),
+          transaction_hash: event.input.transactionHash,
+          user_op_hash: event.input.userOpHash,
+          function_name: event.input.functionName,
+          retry_count: event.input.retryCount,
+          rpc_provider: event.input.provider,
+          ms_since_queue: event.input.msSinceQueue,
+          ms_since_send: event.input.msSinceSend,
+          onchain_status: event.input.onchainStatus,
+        })),
+      );
+    }
+  } catch (error) {
+    logger({
+      service: "server",
+      level: "error",
+      message: "reportUsage error",
+      error,
+    });
   }
+}
 
-  usageEvents.map(async ({ action, input, error }) => {
-    try {
-      const requestBody: UsageEvent = {
-        source: "engine",
-        action,
-        clientId: thirdwebClientId,
-        chainId: input.chainId,
-        walletAddress: input.from,
-        contractAddress: input.to,
-        transactionValue: input.value?.toString(),
-        transactionHash: input.transactionHash,
-        userOpHash: input.userOpHash,
-        errorCode: input.onChainTxStatus === 0 ? "EXECUTION_REVERTED" : error,
-        functionName: input.functionName,
-        extension: input.extension,
-        retryCount: input.retryCount,
-        provider: input.provider,
-        msSinceQueue: input.msSinceQueue,
-        msSinceSend: input.msSinceSend,
-      };
+// Generic usageV1 helper.
+async function reportUsageV1(events: UsageEvent[]) {
+  const BATCH_SIZE = 20;
+  for (let i = 0; i < events.length; i += BATCH_SIZE) {
+    const batch = events.slice(i, i + BATCH_SIZE);
 
-      fetch(env.CLIENT_ANALYTICS_URL, {
-        method: "POST",
-        headers: ANALYTICS_DEFAULT_HEADERS,
-        body: JSON.stringify(requestBody),
-      }).catch(() => {}); // Catch uncaught exceptions since this fetch call is non-blocking.
-    } catch (e) {
-      logger({
-        service: "worker",
-        level: "error",
-        message: "Error:",
-        error: e,
-      });
-    }
+    await Promise.allSettled(
+      batch.map(async (event) => {
+        await fetch(env.CLIENT_ANALYTICS_URL, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "x-sdk-version": process.env.ENGINE_VERSION ?? "",
+            "x-product-name": "engine",
+            "x-client-id": thirdwebClientId,
+          },
+          body: JSON.stringify(event),
+        });
+      }),
+    );
+  }
+}
+
+// Generic usageV2 helper.
+async function reportUsageV2(events: ClientUsageV2Event[]) {
+  await sendUsageV2Events(events, {
+    environment: env.NODE_ENV === "production" ? "production" : "development",
+    source: "engine",
+    thirdwebSecretKey: env.THIRDWEB_API_SECRET_KEY,
   });
-};
+}
diff --git a/yarn.lock b/yarn.lock
@@ -3983,13 +3983,15 @@
     yaml "^2.4.1"
     zod "^3.22.4"
 
-"@thirdweb-dev/service-utils@^0.4.28":
-  version "0.4.52"
-  resolved "https://registry.yarnpkg.com/@thirdweb-dev/service-utils/-/service-utils-0.4.52.tgz#6473ab0fabf7e9ee138776855c5bb4729f88efdb"
-  integrity sha512-24g+juzurNBRQ4llwQBbfQE/msQplFTbnDsp2ihxkb9KThTwI5+d/jWvEoXumFDWm3Ji+FDBJIbNaY9H2Fz6AQ==
+"@thirdweb-dev/service-utils@^0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117":
+  version "0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117"
+  resolved "https://registry.yarnpkg.com/@thirdweb-dev/service-utils/-/service-utils-0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117.tgz#b109e191e2e685977809447e7a6d80ca99594526"
+  integrity sha512-NZHxg/lBipF3dLl3P8HMA4FASdOLjmyAXQFlsFFaWpP9LgawPwauwzLOD79Ym+MnpHVbIFn30pZbus9Ob5WTqg==
   dependencies:
     aws4fetch "1.0.20"
-    zod "3.23.8"
+    kafkajs "2.2.4"
+    lz4js "0.2.0"
+    zod "3.24.1"
 
 "@thirdweb-dev/storage@2.0.15":
   version "2.0.15"
@@ -7972,6 +7974,11 @@ jws@^4.0.0:
     jwa "^2.0.0"
     safe-buffer "^5.0.1"
 
+kafkajs@2.2.4:
+  version "2.2.4"
+  resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.2.4.tgz#59e6e16459d87fdf8b64be73970ed5aa42370a5b"
+  integrity sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==
+
 keccak@^3.0.0, keccak@^3.0.2, keccak@^3.0.3:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/keccak/-/keccak-3.0.4.tgz#edc09b89e633c0549da444432ecf062ffadee86d"
@@ -8260,6 +8267,11 @@ luxon@^3.2.1:
   resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.5.0.tgz#6b6f65c5cd1d61d1fd19dbf07ee87a50bf4b8e20"
   integrity sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==
 
+lz4js@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/lz4js/-/lz4js-0.2.0.tgz#09f1a397cb2158f675146c3351dde85058cb322f"
+  integrity sha512-gY2Ia9Lm7Ep8qMiuGRhvUq0Q7qUereeldZPP1PMEJxPtEWHJLqw9pgX68oHajBH0nzJK4MaZEA/YNV3jT8u8Bg==
+
 magic-sdk@^13.6.2:
   version "13.6.2"
   resolved "https://registry.yarnpkg.com/magic-sdk/-/magic-sdk-13.6.2.tgz#68766fd9d1805332d2a00e5da1bd30fce251a6ac"
@@ -11357,12 +11369,7 @@ zod-to-json-schema@^3.23.0:
   resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.24.1.tgz#f08c6725091aadabffa820ba8d50c7ab527f227a"
   integrity sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w==
 
-zod@3.23.8:
-  version "3.23.8"
-  resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d"
-  integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==
-
-zod@^3.21.4, zod@^3.22.4, zod@^3.23.8:
+zod@3.24.1, zod@^3.21.4, zod@^3.22.4, zod@^3.23.8:
   version "3.24.1"
   resolved "https://registry.yarnpkg.com/zod/-/zod-3.24.1.tgz#27445c912738c8ad1e9de1bea0359fa44d9d35ee"
   integrity sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==