Managed difference in AWS auth between local and AWS with an environment variable

Author: cpcundill

docker-compose.yml

@@ -34,6 +34,12 @@ services:
       DUCKDB_S3_USE_SSL: false
       AWS_REGION: eu-west-2
       LOGGING_LEVEL: DEBUG
+      USE_AWS_CREDENTIAL_CHAIN: false
+
+#      AWS_ACCESS_KEY_ID: "***"
+#      AWS_SECRET_ACCESS_KEY: "***"
+#      AWS_SESSION_TOKEN: "***"
+#      AWS_CREDENTIAL_EXPIRATION: 2024-11-12T10:51:26Z
     restart: on-failure
     ports:
       - "8000:8000"

src/db.py

@@ -6,6 +6,7 @@
 
 collection_bucket = os.environ.get("COLLECTION_BUCKET", "local-collection-data")
 issues_base_path = os.environ.get("ISSUES_BASE_PATH", 'log/issue')
+use_aws_credential_chain = os.environ.get("USE_AWS_CREDENTIAL_CHAIN", 'true').lower() == "true"
 
 logger = get_logger(__name__)
 
@@ -31,8 +32,9 @@ def search_issues(params: IssuesParams):
 
     with duckdb.connect() as conn:
         try:
-            logger.info(conn.execute("CREATE SECRET aws (TYPE S3, PROVIDER CREDENTIAL_CHAIN);").fetchall())
-            logger.info(conn.execute("FROM duckdb_secrets();").fetchall())
+            if use_aws_credential_chain:
+                logger.debug(conn.execute("CREATE SECRET aws (TYPE S3, PROVIDER CREDENTIAL_CHAIN);").fetchall())
+                logger.debug(conn.execute("FROM duckdb_secrets();").fetchall())
             count = conn.execute(sql_count).fetchone()[0]  # Count is first item in Tuple
             logger.debug(count)
             results = conn.execute(sql_results).arrow().to_pylist()

src/schema.py

@@ -7,7 +7,7 @@
 class IssuesParams(BaseModel):
     offset: int = Field(0, ge=0)
     limit: int = Field(50, ge=1, le=100)
-    dataset: Optional[str] = Field(None)
+    dataset: str
     resource: Optional[str] = Field(None)
     field: Optional[str] = Field(None)
     issue_type: Optional[str] = Field(None)