Skip to content

Commit a8d3344

Browse files
ssadhu-slssadhu-sl
committed
Adding test (DRAFT version)
1 parent aa5e14e commit a8d3344

19 files changed

+721
-15
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
.DS_Store
22
.idea
3+
.venv
34
localstack/volume

‎Dockerfile

+5
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,16 @@ ENV GIT_COMMIT=$GIT_COMMIT
77
# RUN apk add --no-cache libcurl curl-dev build-base linux-headers libffi-dev
88

99
COPY requirements/requirements.txt requirements/requirements.txt
10+
COPY requirements/test_requirements.txt requirements/test_requirements.txt
1011

1112
RUN python -m pip install -r requirements/requirements.txt
1213

14+
RUN python -m pip install -r requirements/test_requirements.txt
15+
1316
COPY src/. .
1417

18+
COPY tests/. tests/.
19+
1520
COPY docker-entrypoint.sh docker-entrypoint.sh
1621

1722
# COPY request-api/makefile makefile

‎Makefile

+22-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,23 @@
11
compose-up:
2-
docker compose up -d --build
2+
docker compose up -d --build
3+
4+
test: lint test-coverage
5+
6+
test-coverage:: coverage-unit coverage-integration
7+
8+
coverage-unit:
9+
pytest --cov=src tests/unit/
10+
11+
coverage-integration:
12+
pytest --cov=src --cov-append --cov-fail-under=70 tests/integration/
13+
14+
lint:: black-check flake8
15+
16+
black-check:
17+
black --check .
18+
19+
black:
20+
black .
21+
22+
flake8:
23+
flake8 .

‎docker-compose.yml

+20-3
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ services:
3131
AWS_DEFAULT_REGION: eu-west-2
3232
AWS_ENDPOINT_URL: http://localstack:4566
3333
DUCKDB_S3_ENDPOINT: s3.localhost.localstack.cloud:4566
34-
DUCKDB_S3_USE_SSL: false
34+
DUCKDB_S3_USE_SSL: "false"
3535
AWS_REGION: eu-west-2
3636
LOGGING_LEVEL: DEBUG
37-
USE_AWS_CREDENTIAL_CHAIN: false
37+
USE_AWS_CREDENTIAL_CHAIN: "false"
3838

3939
# AWS_ACCESS_KEY_ID: "***"
4040
# AWS_SECRET_ACCESS_KEY: "***"
@@ -48,7 +48,24 @@ services:
4848
- 10.0.2.20
4949
networks:
5050
- ls
51-
51+
test:
52+
build:
53+
context: .
54+
dockerfile: Dockerfile
55+
environment:
56+
TEST_ENV: "true"
57+
AWS_ACCESS_KEY_ID: testing
58+
AWS_SECRET_ACCESS_KEY: testing
59+
AWS_DEFAULT_REGION: us-west-2
60+
AWS_ENDPOINT_URL: http://localstack:4566
61+
COLLECTION_BUCKET: test-bucket
62+
LOGGING_LEVEL: DEBUG
63+
USE_AWS_CREDENTIAL_CHAIN: "false"
64+
command: ["pytest", "tests/"]
65+
depends_on:
66+
- localstack
67+
networks:
68+
- ls
5269
networks:
5370
# localstack network is used so that requests to localstack resolve from containers
5471
# See https://docs.localstack.cloud/references/network-troubleshooting/endpoint-url/

‎issues/issues.parquet

2.86 KB
Binary file not shown.

‎pytest.ini

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[pytest]
2+
pythonpath = src

‎requirements/test_requirements.in

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-r requirements.txt
2+
pytest
3+
moto[s3]
4+
testcontainers[localstack]

‎requirements/test_requirements.txt

+222
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.10
3+
# by the following command:
4+
#
5+
# pip-compile requirements/test_requirements.in
6+
#
7+
annotated-types==0.7.0
8+
# via
9+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
10+
# pydantic
11+
anyio==4.6.2.post1
12+
# via
13+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
14+
# httpx
15+
# starlette
16+
# watchfiles
17+
boto3==1.35.73
18+
# via moto
19+
botocore==1.35.73
20+
# via
21+
# boto3
22+
# moto
23+
# s3transfer
24+
certifi==2024.8.30
25+
# via
26+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
27+
# httpcore
28+
# httpx
29+
# requests
30+
cffi==1.17.1
31+
# via cryptography
32+
charset-normalizer==3.4.0
33+
# via requests
34+
click==8.1.7
35+
# via
36+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
37+
# typer
38+
# uvicorn
39+
cryptography==44.0.0
40+
# via moto
41+
dnspython==2.7.0
42+
# via
43+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
44+
# email-validator
45+
duckdb==1.1.3
46+
# via -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
47+
email-validator==2.2.0
48+
# via
49+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
50+
# fastapi
51+
exceptiongroup==1.2.2
52+
# via
53+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
54+
# anyio
55+
# pytest
56+
fastapi[standard]==0.115.4
57+
# via -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
58+
fastapi-cli[standard]==0.0.5
59+
# via
60+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
61+
# fastapi
62+
h11==0.14.0
63+
# via
64+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
65+
# httpcore
66+
# uvicorn
67+
httpcore==1.0.6
68+
# via
69+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
70+
# httpx
71+
httptools==0.6.4
72+
# via
73+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
74+
# uvicorn
75+
httpx==0.27.2
76+
# via
77+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
78+
# fastapi
79+
idna==3.10
80+
# via
81+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
82+
# anyio
83+
# email-validator
84+
# httpx
85+
# requests
86+
iniconfig==2.0.0
87+
# via pytest
88+
jinja2==3.1.4
89+
# via
90+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
91+
# fastapi
92+
# moto
93+
jmespath==1.0.1
94+
# via
95+
# boto3
96+
# botocore
97+
markdown-it-py==3.0.0
98+
# via
99+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
100+
# rich
101+
markupsafe==3.0.2
102+
# via
103+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
104+
# jinja2
105+
# werkzeug
106+
mdurl==0.1.2
107+
# via
108+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
109+
# markdown-it-py
110+
moto[s3]==5.0.22
111+
# via -r requirements/test_requirements.in
112+
packaging==24.2
113+
# via pytest
114+
pluggy==1.5.0
115+
# via pytest
116+
py-partiql-parser==0.5.6
117+
# via moto
118+
pyarrow==18.0.0
119+
# via -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
120+
pycparser==2.22
121+
# via cffi
122+
pydantic==2.9.2
123+
# via
124+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
125+
# fastapi
126+
pydantic-core==2.23.4
127+
# via
128+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
129+
# pydantic
130+
pygments==2.18.0
131+
# via
132+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
133+
# rich
134+
pytest==8.3.4
135+
# via -r requirements/test_requirements.in
136+
python-dateutil==2.9.0.post0
137+
# via
138+
# botocore
139+
# moto
140+
python-dotenv==1.0.1
141+
# via
142+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
143+
# uvicorn
144+
python-multipart==0.0.17
145+
# via
146+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
147+
# fastapi
148+
pyyaml==6.0.2
149+
# via
150+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
151+
# moto
152+
# responses
153+
# uvicorn
154+
requests==2.32.3
155+
# via
156+
# moto
157+
# responses
158+
responses==0.25.3
159+
# via moto
160+
rich==13.9.4
161+
# via
162+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
163+
# typer
164+
s3transfer==0.10.4
165+
# via boto3
166+
shellingham==1.5.4
167+
# via
168+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
169+
# typer
170+
six==1.16.0
171+
# via python-dateutil
172+
sniffio==1.3.1
173+
# via
174+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
175+
# anyio
176+
# httpx
177+
starlette==0.41.2
178+
# via
179+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
180+
# fastapi
181+
tomli==2.2.1
182+
# via pytest
183+
typer==0.13.0
184+
# via
185+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
186+
# fastapi-cli
187+
typing-extensions==4.12.2
188+
# via
189+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
190+
# anyio
191+
# fastapi
192+
# pydantic
193+
# pydantic-core
194+
# rich
195+
# typer
196+
# uvicorn
197+
urllib3==1.26.18
198+
# via
199+
# botocore
200+
# requests
201+
# responses
202+
uvicorn[standard]==0.32.0
203+
# via
204+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
205+
# fastapi
206+
# fastapi-cli
207+
uvloop==0.21.0
208+
# via
209+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
210+
# uvicorn
211+
watchfiles==0.24.0
212+
# via
213+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
214+
# uvicorn
215+
websockets==13.1
216+
# via
217+
# -r /home/ssadhu/pipeline-internal-api/requirements/requirements.txt
218+
# uvicorn
219+
werkzeug==3.1.3
220+
# via moto
221+
xmltodict==0.14.2
222+
# via moto

‎src/__init__.py

Whitespace-only changes.

‎src/config.py

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# config.py
2+
import os
3+
4+
5+
class config:
6+
"""Centralized configuration management."""
7+
collection_bucket = os.environ.get("COLLECTION_BUCKET", "local-collection-data")
8+
issues_base_path = os.environ.get("ISSUES_BASE_PATH", 'log/issue')
9+
use_aws_credential_chain = os.environ.get("USE_AWS_CREDENTIAL_CHAIN", 'true').lower() == "true"

‎src/db.py

+4-5
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,15 @@
33
from log import get_logger
44
from schema import IssuesParams
55
from pagination_model import PaginationParams, PaginatedResult
6+
from config import config
67

7-
collection_bucket = os.environ.get("COLLECTION_BUCKET", "local-collection-data")
8-
issues_base_path = os.environ.get("ISSUES_BASE_PATH", 'log/issue')
9-
use_aws_credential_chain = os.environ.get("USE_AWS_CREDENTIAL_CHAIN", 'true').lower() == "true"
108

119
logger = get_logger(__name__)
1210

1311

1412
def search_issues(params: IssuesParams):
15-
s3_uri = f"s3://{collection_bucket}/{issues_base_path}/**/*.parquet"
13+
s3_uri = f"s3://{config.collection_bucket}/{config.issues_base_path}/**/*.parquet"
14+
1615
pagination = f"LIMIT {params.limit} OFFSET {params.offset}"
1716

1817
where_clause = ""
@@ -32,7 +31,7 @@ def search_issues(params: IssuesParams):
3231

3332
with duckdb.connect() as conn:
3433
try:
35-
if use_aws_credential_chain:
34+
if config.use_aws_credential_chain:
3635
logger.debug(conn.execute("CREATE SECRET aws (TYPE S3, PROVIDER CREDENTIAL_CHAIN);").fetchall())
3736
logger.debug(conn.execute("FROM duckdb_secrets();").fetchall())
3837
count = conn.execute(sql_count).fetchone()[0] # Count is first item in Tuple

0 commit comments

Comments
 (0)