Nabu batch size (#120)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
internetofwater · Mar 4, 2025 · 7c2d55c · 7c2d55c
1 parent 5158088
commit 7c2d55c
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 1 deletion.
diff --git a/.env.example b/.env.example
@@ -17,6 +17,7 @@ GLEANER_THREADS=5
 NABU_PROFILING=false
 # log levels should be in all caps
 NABU_LOG_LEVEL=INFO
+NABU_BATCH_SIZE=100
 
 
 # Minio

diff --git a/userCode/lib/containers.py b/userCode/lib/containers.py
@@ -4,7 +4,7 @@
 import os
 from pathlib import Path
 from userCode.lib.types import cli_modes
-from userCode.lib.env import GLEANER_IMAGE, NABU_IMAGE, NABU_PROFILING
+from userCode.lib.env import GLEANER_IMAGE, NABU_BATCH_SIZE, NABU_IMAGE, NABU_PROFILING
 from userCode.lib.utils import run_scheduler_docker_image
 
 
@@ -43,6 +43,8 @@ def run(self, args: list[str]):
         if NABU_PROFILING:
             args.append("--trace")
 
+        args.append(f"--upsert-batch-size={NABU_BATCH_SIZE}")
+
         nabu_log_level = os.environ.get("NABU_LOG_LEVEL")
         if nabu_log_level:
             args.append(f"--log-level={nabu_log_level}")

diff --git a/userCode/lib/env.py b/userCode/lib/env.py
@@ -110,3 +110,4 @@ def strict_env(key: str):
 
 
 NABU_PROFILING = strict_env("NABU_PROFILING")
+NABU_BATCH_SIZE = strict_env_int("NABU_BATCH_SIZE")
Original file line number	Diff line number	Diff line change
Expand Up		@@ -110,3 +110,4 @@ def strict_env(key: str):


		NABU_PROFILING = strict_env("NABU_PROFILING")
		NABU_BATCH_SIZE = strict_env_int("NABU_BATCH_SIZE")