Adding integration tests

Author: ssadhu-sl

src/db.py

@@ -139,7 +139,7 @@ def search_issue_type_summary(params: IssueTypeSummaryParams):
 
     sql_count = f"SELECT COUNT(*) FROM '{s3_uri}' {where_clause}"
     sql_results = f"""
-    SELECT organisation, organisation_name, dataset, issue_type, fields, 
+    SELECT organisation, organisation_name, dataset, resource, issue_type, field, 
 count_issues, severity, responsibility  FROM '{s3_uri}' {where_clause} LIMIT ? OFFSET ?"""
 
     logger.debug(sql_count)
@@ -188,9 +188,7 @@ def search_dataset_resource_mapping(params: CommonParams):
         query_params.append(params.organisation)
 
     sql_count = f"SELECT COUNT(*) FROM '{s3_uri}' {where_clause}"
-    sql_results = f"""
-    SELECT organisation, dataset, resource, mapped_fields FROM 
-'{s3_uri}' {where_clause} LIMIT ? OFFSET ?"""
+    sql_results = f"SELECT * FROM '{s3_uri}' {where_clause} LIMIT ? OFFSET ?"
 
     logger.debug(sql_count)
     logger.debug(sql_results)

tests/conftest.py

@@ -68,13 +68,17 @@ def s3_bucket(s3_client, test_dir):
             Body=f,
         )
 
-    parquet_file1 = f"{test_dir}/../files/provision_summary.parquet"
-    with open(parquet_file1, "rb") as f:
-        s3_client.put_object(
-            Bucket=bucket_name,
-            Key=f"{os.environ['PERFORMANCE_BASE_PATH']}/provision_summary.parquet",
-            Body=f,
-        )
+    perf_parquet_files = glob.glob(f"{test_dir}/../files/performance/*.parquet")
+    for file_path in perf_parquet_files:
+        file_name = os.path.basename(file_path)
+        s3_key = f"{os.environ['PERFORMANCE_BASE_PATH']}/{file_name}"
+
+        with open(file_path, "rb") as f:
+            s3_client.put_object(
+                Bucket=bucket_name,
+                Key=s3_key,
+                Body=f,
+            )
 
     parquet_file2 = f"{test_dir}/../files/specification.parquet"
     with open(parquet_file2, "rb") as f:

tests/files/performance/endpoint_dataset_issue_type_summary.parquet

@@ -1,6 +1,5 @@
 from fastapi.testclient import TestClient
 from main import app
-from unittest.mock import patch
 import json
 
 # Create a test client for the FastAPI app
@@ -15,7 +14,6 @@ def test_search_issues(s3_bucket):
         "limit": 10,
     }
 
-    # Test the function that interacts with DuckDB and S3 via LocalStack
     response = client.get("/log/issue", params=params)
 
     # Validate the results from the search
@@ -75,15 +73,14 @@ def test_provision_summary(s3_bucket):
             ), "Expected active endpoint count to be 1"
 
 
-def test_specification(s3_bucket, duckdb_connection):
+def test_specification(s3_bucket):
     # Prepare test params
     params = {
         "offset": 0,
         "limit": 8,
     }
 
-    with patch("db.duckdb.connect", return_value=duckdb_connection):
-        response = client.get("/specification/specification", params=params)
+    response = client.get("/specification/specification", params=params)
 
     # Validate the results from the search
     assert response.status_code == 200
@@ -96,16 +93,15 @@ def test_specification(s3_bucket, duckdb_connection):
     assert len(response_data) > 0
 
 
-def test_specification_with_dataset(s3_bucket, duckdb_connection):
+def test_specification_with_dataset(s3_bucket):
     # Prepare test params
     params = {
         "offset": 0,
         "limit": 8,
         "dataset": "article-4-direction-area",
     }
 
-    with patch("db.duckdb.connect", return_value=duckdb_connection):
-        response = client.get("/specification/specification", params=params)
+    response = client.get("/specification/specification", params=params)
 
     # Validate the results from the search
     assert response.status_code == 200
@@ -138,7 +134,6 @@ def test_issue_type_summary(s3_bucket):
     assert "X-Pagination-Total-Results" in response.headers
     assert response.headers["X-Pagination-Total-Results"] == str(11)
     assert response.headers["X-Pagination-Limit"] == "8"
-
     assert len(response_data) > 0
     filtered_rows = [
         item
@@ -147,7 +142,62 @@ def test_issue_type_summary(s3_bucket):
         == "8c61c7b72902daeaaa462002e62d840ce3916defacd54db97986654b180ce250"
     ]
 
-    assert len(filtered_rows) == 5
+    assert len(filtered_rows) == 6
+    assert sum(1 for item in filtered_rows if item.get("issue_type") == "patch") == 4
+    assert (
+        sum(
+            item.get("count_issues")
+            for item in filtered_rows
+            if item.get("issue_type") == "patch"
+        )
+        == 807
+    )
+
+
+def test_dataset_resource_mapping(s3_bucket):
+    # Prepare test params
+    params = {
+        "organisation": "local-authority:BUC",
+        "dataset": "article-4-direction-area",
+        "offset": 0,
+        "limit": 8,
+    }
+    response = client.get("/performance/dataset_resource_mapping", params=params)
+
+    # Validate the results from the search
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "X-Pagination-Total-Results" in response.headers
+    assert response.headers["X-Pagination-Total-Results"] == str(9)
+    assert response.headers["X-Pagination-Limit"] == "8"
+    assert len(response_data) > 0
+    assert response_data[0]["mapping_field"] == "geometry"
+    assert (
+        response_data[0]["non_mapping_field"]
+        == "start-date;entry-date;name;description;reference"
+    )
+
+
+def test_endpoint_dataset_summary(s3_bucket):
+    # Prepare test params
+    params = {
+        "organisation": "local-authority:BUC",
+        "dataset": "article-4-direction-area",
+        "offset": 0,
+        "limit": 10,
+    }
+    response = client.get("/performance/endpoint_dataset_summary", params=params)
+
+    # Validate the results from the search
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "X-Pagination-Total-Results" in response.headers
+    assert response.headers["X-Pagination-Total-Results"] == str(7)
+    assert response.headers["X-Pagination-Limit"] == "10"
+    assert len(response_data) > 0
     assert (
-        sum(1 for item in filtered_rows if item.get("issue_type") == "patch") == 3
-    )  # Ensure 3 have issue_type as "patch"
+        response_data[0]["endpoint"]
+        == "01de81578391eab3a1fdc9fcb92e559aa0a0bf1aafc777905d680fc0e189811e"
+    )