Adding issue-type_summary API

ssadhu-sl · ssadhu-sl · commit 37669f08c743 · 2025-03-04T15:09:05.000Z
diff --git a/localstack/bootstrap/collection-data/data/performance/endpoint_dataset_issue_type_summary.parquet b/localstack/bootstrap/collection-data/data/performance/endpoint_dataset_issue_type_summary.parquet
diff --git a/localstack/bootstrap/collection-data/data/performance/endpoint_dataset_resource_summary.parquet b/localstack/bootstrap/collection-data/data/performance/endpoint_dataset_resource_summary.parquet
diff --git a/src/db.py b/src/db.py
@@ -1,6 +1,11 @@
 import duckdb
 from log import get_logger
-from schema import IssuesParams, ProvisionParams, SpecificationsParams
+from schema import (
+    CommonParams,
+    IssuesParams,
+    SpecificationsParams,
+    IssueTypeSummaryParams,
+)
 from pagination_model import PaginationParams, PaginatedResult
 from config import config
 import json
@@ -54,8 +59,174 @@ def search_issues(params: IssuesParams):
             raise e
 
 
-def search_provision_summary(params: ProvisionParams):
-    s3_uri = f"s3://{config.collection_bucket}/{config.performance_base_path}/*.parquet"  # noqa
+def search_provision_summary(params: CommonParams):
+    s3_uri = f"s3://{config.collection_bucket}/{config.performance_base_path}/provision_summary.parquet"  # noqa
+
+    where_clause = ""
+    query_params = []
+
+    if params.dataset:
+        where_clause += _add_condition(where_clause, "dataset = ?")
+        query_params.append(params.dataset)
+
+    if params.organisation:
+        where_clause += _add_condition(where_clause, "organisation = ?")
+        query_params.append(params.organisation)
+
+    sql_count = f"SELECT COUNT(*) FROM '{s3_uri}' {where_clause}"
+    sql_results = f"SELECT * FROM '{s3_uri}' {where_clause} LIMIT ? OFFSET ?"
+
+    logger.debug(sql_count)
+    logger.debug(sql_results)
+
+    with duckdb.connect() as conn:
+        try:
+            if config.use_aws_credential_chain:
+                logger.debug(
+                    conn.execute(
+                        "CREATE SECRET aws (TYPE S3, PROVIDER CREDENTIAL_CHAIN);"
+                    ).fetchall()
+                )
+                logger.debug(conn.execute("FROM duckdb_secrets();").fetchall())
+
+            # Execute parameterized queries
+            count = conn.execute(sql_count, query_params).fetchone()[0]
+            results = (
+                conn.execute(sql_results, query_params + [params.limit, params.offset])
+                .arrow()
+                .to_pylist()
+            )
+
+            return PaginatedResult(
+                params=PaginationParams(offset=params.offset, limit=params.limit),
+                total_results_available=count,
+                data=results,
+            )
+        except Exception as e:
+            logger.exception("Failure executing DuckDB queries")
+            raise e
+
+
+def search_issue_type_summary(params: IssueTypeSummaryParams):
+    s3_uri = f"s3://{config.collection_bucket}/{config.performance_base_path}/endpoint_dataset_issue_type_summary.parquet"  # noqa
+
+    where_clause = ""
+    query_params = []
+
+    if params.dataset:
+        where_clause += _add_condition(where_clause, "dataset = ?")
+        query_params.append(params.dataset)
+
+    if params.organisation:
+        where_clause += _add_condition(where_clause, "organisation = ?")
+        query_params.append(params.organisation)
+
+    if params.issueType:
+        where_clause += _add_condition(where_clause, "issue_type = ?")
+        query_params.append(params.issueType)
+
+    if params.issueField:
+        where_clause += _add_condition(where_clause, "issue_field = ?")
+        query_params.append(params.issueField)
+
+    if params.severity:
+        where_clause += _add_condition(where_clause, "severity = ?")
+        query_params.append(params.severity)
+
+    if params.responsibility:
+        where_clause += _add_condition(where_clause, "responsibility = ?")
+        query_params.append(params.responsibility)
+
+    sql_count = f"SELECT COUNT(*) FROM '{s3_uri}' {where_clause}"
+    sql_results = f"""
+        SELECT organisation, organisation_name, dataset, issue_type, fields,
+    count_issues, severity, responsibility  FROM '{s3_uri}' {where_clause} 
+    LIMIT ? OFFSET ?"""
+
+    logger.debug(sql_count)
+    logger.debug(sql_results)
+
+    with duckdb.connect() as conn:
+        try:
+            if config.use_aws_credential_chain:
+                logger.debug(
+                    conn.execute(
+                        "CREATE SECRET aws (TYPE S3, PROVIDER CREDENTIAL_CHAIN);"
+                    ).fetchall()
+                )
+                logger.debug(conn.execute("FROM duckdb_secrets();").fetchall())
+
+            # Execute parameterized queries
+            count = conn.execute(sql_count, query_params).fetchone()[0]
+            results = (
+                conn.execute(sql_results, query_params + [params.limit, params.offset])
+                .arrow()
+                .to_pylist()
+            )
+
+            return PaginatedResult(
+                params=PaginationParams(offset=params.offset, limit=params.limit),
+                total_results_available=count,
+                data=results,
+            )
+        except Exception as e:
+            logger.exception("Failure executing DuckDB queries")
+            raise e
+
+
+def search_dataset_resource_mapping(params: CommonParams):
+    s3_uri = f"s3://{config.collection_bucket}/{config.performance_base_path}/endpoint_dataset_resource_summary.parquet"  # noqa
+
+    where_clause = ""
+    query_params = []
+
+    if params.dataset:
+        where_clause += _add_condition(where_clause, "dataset = ?")
+        query_params.append(params.dataset)
+
+    if params.organisation:
+        where_clause += _add_condition(where_clause, "organisation = ?")
+        query_params.append(params.organisation)
+
+    sql_count = f"SELECT COUNT(*) FROM '{s3_uri}' {where_clause}"
+    sql_results = f"""
+        SELECT organisation, dataset, resource, mapped_fields FROM 
+        '{s3_uri}' {where_clause} 
+        LIMIT ? OFFSET ?"""
+
+    logger.debug(sql_count)
+    logger.debug(sql_results)
+
+    with duckdb.connect() as conn:
+        try:
+            if config.use_aws_credential_chain:
+                logger.debug(
+                    conn.execute(
+                        "CREATE SECRET aws (TYPE S3, PROVIDER CREDENTIAL_CHAIN);"
+                    ).fetchall()
+                )
+                logger.debug(conn.execute("FROM duckdb_secrets();").fetchall())
+
+            # Execute parameterized queries
+            count = conn.execute(sql_count, query_params).fetchone()[0]
+            results = (
+                conn.execute(sql_results, query_params + [params.limit, params.offset])
+                .arrow()
+                .to_pylist()
+            )
+
+            return PaginatedResult(
+                params=PaginationParams(offset=params.offset, limit=params.limit),
+                total_results_available=count,
+                data=results,
+            )
+        except Exception as e:
+            logger.exception("Failure executing DuckDB queries")
+            raise e
+
+
+def search_endpoint_dataset_summary(params: CommonParams):
+    s3_uri = f"s3://{config.collection_bucket}/{config.performance_base_path}/endpoint_dataset_summary.parquet"  # noqa
 
     where_clause = ""
     query_params = []
diff --git a/src/main.py b/src/main.py
@@ -8,9 +8,10 @@
 from fastapi.responses import JSONResponse
 from schema import (
     HealthCheckResponse,
+    CommonParams,
     IssuesParams,
-    ProvisionParams,
     SpecificationsParams,
+    IssueTypeSummaryParams,
 )
 from log import get_logger
 
@@ -58,7 +59,7 @@ def issues(http_response: Response, params: IssuesParams = Depends()):
 
 
 @app.get("/performance/provision_summary", tags=["provision_summary"])
-def provision_summary(http_response: Response, params: ProvisionParams = Depends()):
+def provision_summary(http_response: Response, params: CommonParams = Depends()):
     paginated_result = db.search_provision_summary(params)
     http_response.headers["X-Pagination-Total-Results"] = str(
         paginated_result.total_results_available
@@ -72,6 +73,53 @@ def provision_summary(http_response: Response, params: ProvisionParams = Depends
     )
 
 
+@app.get("/performance/issue_type_summary", tags=["issue_type_summary"])
+def issue_type_summary(
+    http_response: Response, params: IssueTypeSummaryParams = Depends()
+):
+    paginated_result = db.search_issue_type_summary(params)
+    http_response.headers["X-Pagination-Total-Results"] = str(
+        paginated_result.total_results_available
+    )
+    http_response.headers["X-Pagination-Offset"] = str(paginated_result.params.offset)
+    http_response.headers["X-Pagination-Limit"] = str(paginated_result.params.limit)
+    return Response(
+        content=json.dumps(paginated_result.data),
+        media_type="application/json",
+        headers=http_response.headers,
+    )
+
+
+@app.get("/performance/dataset_resource_mapping", tags=["dataset_resource_mapping"])
+def dataset_resource_mapping(http_response: Response, params: CommonParams = Depends()):
+    paginated_result = db.search_dataset_resource_mapping(params)
+    http_response.headers["X-Pagination-Total-Results"] = str(
+        paginated_result.total_results_available
+    )
+    http_response.headers["X-Pagination-Offset"] = str(paginated_result.params.offset)
+    http_response.headers["X-Pagination-Limit"] = str(paginated_result.params.limit)
+    return Response(
+        content=json.dumps(paginated_result.data),
+        media_type="application/json",
+        headers=http_response.headers,
+    )
+
+
+@app.get("/performance/endpoint_dataset_summary", tags=["endpoint_dataset_summary"])
+def endpoint_dataset_summary(http_response: Response, params: CommonParams = Depends()):
+    paginated_result = db.search_endpoint_dataset_summary(params)
+    http_response.headers["X-Pagination-Total-Results"] = str(
+        paginated_result.total_results_available
+    )
+    http_response.headers["X-Pagination-Offset"] = str(paginated_result.params.offset)
+    http_response.headers["X-Pagination-Limit"] = str(paginated_result.params.limit)
+    return Response(
+        content=json.dumps(paginated_result.data),
+        media_type="application/json",
+        headers=http_response.headers,
+    )
+
+
 @app.get("/specification/specification", tags=["specification"])
 def get_specification(
     http_response: Response, params: SpecificationsParams = Depends()
diff --git a/src/schema.py b/src/schema.py
@@ -9,6 +9,11 @@ class BaseParams(BaseModel):
     limit: int = Field(50, ge=1, le=100)
 
 
+class CommonParams(BaseParams):
+    dataset: Optional[str] = Field(None)
+    organisation: Optional[str] = Field(None)
+
+
 class IssuesParams(BaseParams):
     dataset: Optional[str] = Field(None)
     resource: Optional[str] = Field(None)
@@ -37,9 +42,13 @@ class Issue(BaseModel):
     message: str
 
 
-class ProvisionParams(BaseParams):
+class IssueTypeSummaryParams(BaseParams):
     dataset: Optional[str] = Field(None)
     organisation: Optional[str] = Field(None)
+    issueType: Optional[str] = Field(None)
+    issueField: Optional[str] = Field(None)
+    severity: Optional[str] = Field(None)
+    responsibility: Optional[str] = Field(None)
 
 
 class SpecificationsParams(BaseParams):
diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py
@@ -1,5 +1,6 @@
 from fastapi.testclient import TestClient
 from main import app
+from unittest.mock import patch
 import json
 
 # Create a test client for the FastAPI app
@@ -74,14 +75,15 @@ def test_provision_summary(s3_bucket):
             ), "Expected active endpoint count to be 1"
 
 
-def test_specification(s3_bucket):
+def test_specification(s3_bucket, duckdb_connection):
     # Prepare test params
     params = {
         "offset": 0,
         "limit": 8,
     }
 
-    response = client.get("/specification/specification", params=params)
+    with patch("db.duckdb.connect", return_value=duckdb_connection):
+        response = client.get("/specification/specification", params=params)
 
     # Validate the results from the search
     assert response.status_code == 200
@@ -94,15 +96,16 @@ def test_specification(s3_bucket):
     assert len(response_data) > 0
 
 
-def test_specification_with_dataset(s3_bucket):
+def test_specification_with_dataset(s3_bucket, duckdb_connection):
     # Prepare test params
     params = {
         "offset": 0,
         "limit": 8,
         "dataset": "article-4-direction-area",
     }
 
-    response = client.get("/specification/specification", params=params)
+    with patch("db.duckdb.connect", return_value=duckdb_connection):
+        response = client.get("/specification/specification", params=params)
 
     # Validate the results from the search
     assert response.status_code == 200
@@ -116,3 +119,30 @@ def test_specification_with_dataset(s3_bucket):
     assert response_data[0]["dataset"] == "article-4-direction-area"
     assert response_data[0]["fields"]
     assert len(response_data[0]["fields"]) > 1
+
+def test_issue_type_summary(s3_bucket):
+    # Prepare test params
+    params = {
+        "organisation": "local-authority:BUC",
+        "dataset": "brownfield-land",
+        "offset": 0,
+        "limit": 8,
+    }
+    response = client.get("/performance/issue_type_summary", params=params)
+
+    # Validate the results from the search
+    assert response.status_code == 200
+
+    response_data = response.json()
+    assert "X-Pagination-Total-Results" in response.headers
+    assert response.headers["X-Pagination-Total-Results"] == str(11)
+    assert response.headers["X-Pagination-Limit"] == "8"
+
+    assert len(response_data) > 0
+    filtered_rows = [
+            item for item in response_data
+            if item.get("resource") == "8c61c7b72902daeaaa462002e62d840ce3916defacd54db97986654b180ce250"
+        ]
+    
+    assert len(filtered_rows) == 5 
+    assert sum(1 for item in filtered_rows if item.get("issue_type") == "patch") == 3  # Ensure 3 have issue_type as "patch"
