Merge pull request #403 from aldbr/main_FEAT_add-new-bl-layer

feat: add a business logic layer

.github/workflows/extensions.yml

@@ -27,11 +27,13 @@ jobs:
           - package: "./extensions/gubbins/gubbins-core"
             dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./diracx-core"
           - package: "./extensions/gubbins/gubbins-db"
-            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core "
+            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core"
+          - package: "./extensions/gubbins/gubbins-logic"
+            dependencies: "./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-core ./diracx-db ./diracx-core ./diracx-logic"
           - package: "./extensions/gubbins/gubbins-routers"
-            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core ./diracx-routers"
+            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-logic ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-logic ./diracx-core ./diracx-routers"
           - package: "./extensions/gubbins/gubbins-client"
-            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-client ./diracx-core "
+            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-client ./diracx-core"
           - package: "./extensions/gubbins/gubbins-cli"
             dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-cli ./diracx-client ./diracx-core ./diracx-api"
     steps:
@@ -58,6 +60,7 @@ jobs:
         run: |
           mypy ${{ matrix.package }}/src
       - name: Run pytest
+        if: ${{ matrix.package != './extensions/gubbins/gubbins-logic' }}
         run: |
           cd ${{ matrix.package }}
           pip install .[testing]
@@ -147,7 +150,7 @@ jobs:
           outputs: type=docker,dest=/tmp/gubbins_services_image.tar
           build-args: |
             EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
-            EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
+            EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
       - name: Build and export client
         uses: docker/build-push-action@v6
         with:
@@ -185,7 +188,7 @@ jobs:
         run: |
           pip install pytest-github-actions-annotate-failures
           pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-cli/[testing] ./extensions/gubbins/gubbins-core/[testing]
+          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-logic/[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-cli/[testing] ./extensions/gubbins/gubbins-core/[testing]
       - name: Start demo
         run: |
           git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
@@ -274,7 +277,7 @@ jobs:
         run: |
           micromamba install -c conda-forge nodejs pre-commit
           pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-testing/[testing] -e ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-core/[testing]
+          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-logic/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-logic[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-testing/[testing] -e ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-core/[testing]
           npm install -g autorest
       - name: Run autorest
         run: |

.github/workflows/main.yml

@@ -29,7 +29,6 @@ jobs:
       run: |
         find -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source
 
-
   unittest:
     name: Unit test - ${{ matrix.package }}
     runs-on: ubuntu-latest
@@ -41,8 +40,10 @@ jobs:
             dependencies: "./diracx-testing"
           - package: "./diracx-db"
             dependencies: "./diracx-testing ./diracx-core"
+          - package: "./diracx-logic"
+            dependencies: "./diracx-core ./diracx-db"
           - package: "./diracx-routers"
-            dependencies: "./diracx-testing ./diracx-core ./diracx-db"
+            dependencies: "./diracx-testing ./diracx-core ./diracx-db ./diracx-logic"
           - package: "./diracx-client"
             dependencies: "./diracx-testing ./diracx-core"
           - package: "./diracx-api"
@@ -68,6 +69,7 @@ jobs:
           pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
           pip install ${{ matrix.dependencies }}
       - name: Run pytest
+        if: ${{ matrix.package != './diracx-logic' }}
         run: |
           cd ${{ matrix.package }}
           pip install .[testing]
@@ -93,7 +95,7 @@ jobs:
         run: |
           pip install pytest-github-actions-annotate-failures
           pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/
+          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-db/[testing] ./diracx-testing/
       - name: Start demo
         run: |
           git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
@@ -162,7 +164,7 @@ jobs:
         run: |
           micromamba install -c conda-forge nodejs pre-commit
           pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
-          pip install ./diracx-core/ ./diracx-api/ ./diracx-cli/ -e ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/ ./diracx-testing/
+          pip install ./diracx-core/ ./diracx-api/ ./diracx-cli/ -e ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-logic/[testing] ./diracx-db/ ./diracx-testing/
           npm install -g autorest
       - name: Run autorest
         run: |

diracx-cli/src/diracx/cli/internal/legacy.py

@@ -230,6 +230,7 @@ def generate_helm_values(
     helm_values["diracx"] = diracx_config
     diracx_config["hostname"] = diracx_hostname
 
+    diracx_settings["DIRACX_SERVICE_AUTH_TOKEN_ISSUER"] = diracx_url
     diracx_settings["DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS"] = json.dumps(
         [
             urljoin(diracx_url, "api/docs/oauth2-redirect"),

diracx-core/pyproject.toml

@@ -33,6 +33,7 @@ testing = [
 ]
 types = [
     "botocore-stubs",
+    "types-aiobotocore[essential]",
     "types-aiobotocore-s3",
     "types-cachetools",
     "types-PyYAML",

diracx-core/src/diracx/core/exceptions.py

@@ -28,6 +28,18 @@ class ExpiredFlowError(AuthorizationError):
     """Used only for the Device Flow when the polling is expired."""
 
 
+class IAMServerError(DiracError):
+    """Used whenever we encounter a server problem with the IAM server."""
+
+
+class IAMClientError(DiracError):
+    """Used whenever we encounter a client problem with the IAM server."""
+
+
+class InvalidCredentialsError(DiracError):
+    """Used whenever the credentials are invalid."""
+
+
 class ConfigurationError(DiracError):
     """Used whenever we encounter a problem with the configuration."""
 
@@ -40,6 +52,12 @@ class InvalidQueryError(DiracError):
     """It was not possible to build a valid database query from the given input."""
 
 
+class TokenNotFoundError(Exception):
+    def __init__(self, jti: str, detail: str | None = None):
+        self.jti: str = jti
+        super().__init__(f"Token {jti} not found" + (" ({detail})" if detail else ""))
+
+
 class JobNotFoundError(Exception):
     def __init__(self, job_id: int, detail: str | None = None):
         self.job_id: int = job_id
@@ -66,6 +84,16 @@ def __init__(self, pfn: str, se_name: str, detail: str | None = None):
         )
 
 
+class SandboxAlreadyInsertedError(Exception):
+    def __init__(self, pfn: str, se_name: str, detail: str | None = None):
+        self.pfn: str = pfn
+        self.se_name: str = se_name
+        super().__init__(
+            f"Sandbox with {pfn} and {se_name} already inserted"
+            + (" ({detail})" if detail else "")
+        )
+
+
 class JobError(Exception):
     def __init__(self, job_id, detail: str | None = None):
         self.job_id: int = job_id

diracx-core/src/diracx/core/models.py

@@ -1,3 +1,8 @@
+"""Models are used to define the data structure of the requests and responses
+for the DiracX API. They are shared between the client components (cli, api) and
+services components (db, logic, routers).
+"""
+
 from __future__ import annotations
 
 from datetime import datetime
@@ -46,12 +51,25 @@ class SortSpec(TypedDict):
     direction: SortDirection
 
 
-class TokenResponse(BaseModel):
-    # Based on RFC 6749
-    access_token: str
-    expires_in: int
-    token_type: str = "Bearer"  # noqa: S105
-    refresh_token: str | None = None
+class InsertedJob(TypedDict):
+    JobID: int
+    Status: str
+    MinorStatus: str
+    TimeStamp: datetime
+
+
+class JobSummaryParams(BaseModel):
+    grouping: list[str]
+    search: list[SearchSpec] = []
+    # TODO: Add more validation
+
+
+class JobSearchParams(BaseModel):
+    parameters: list[str] | None = None
+    search: list[SearchSpec] = []
+    sort: list[SortSpec] = []
+    distinct: bool = False
+    # TODO: Add more validation
 
 
 class JobStatus(StrEnum):
@@ -77,6 +95,15 @@ class JobMinorStatus(StrEnum):
     RESCHEDULED = "Job Rescheduled"
 
 
+class JobLoggingRecord(BaseModel):
+    job_id: int
+    status: JobStatus
+    minor_status: str
+    application_status: str
+    date: datetime
+    source: str
+
+
 class JobStatusUpdate(BaseModel):
     Status: JobStatus | None = None
     MinorStatus: str | None = None
@@ -136,3 +163,93 @@ class SandboxInfo(BaseModel):
 class SandboxType(StrEnum):
     Input = "Input"
     Output = "Output"
+
+
+class SandboxDownloadResponse(BaseModel):
+    url: str
+    expires_in: int
+
+
+class SandboxUploadResponse(BaseModel):
+    pfn: str
+    url: str | None = None
+    fields: dict[str, str] = {}
+
+
+class GrantType(StrEnum):
+    """Grant types for OAuth2."""
+
+    authorization_code = "authorization_code"
+    device_code = "urn:ietf:params:oauth:grant-type:device_code"
+    refresh_token = "refresh_token"  # noqa: S105   # False positive of Bandit about hard coded password
+
+
+class InitiateDeviceFlowResponse(TypedDict):
+    """Response for the device flow initiation."""
+
+    user_code: str
+    device_code: str
+    verification_uri_complete: str
+    verification_uri: str
+    expires_in: int
+
+
+class OpenIDConfiguration(TypedDict):
+    issuer: str
+    token_endpoint: str
+    userinfo_endpoint: str
+    authorization_endpoint: str
+    device_authorization_endpoint: str
+    grant_types_supported: list[str]
+    scopes_supported: list[str]
+    response_types_supported: list[str]
+    token_endpoint_auth_signing_alg_values_supported: list[str]
+    token_endpoint_auth_methods_supported: list[str]
+    code_challenge_methods_supported: list[str]
+
+
+class TokenPayload(TypedDict):
+    jti: str
+    exp: datetime
+    dirac_policies: dict
+
+
+class TokenResponse(BaseModel):
+    # Based on RFC 6749
+    access_token: str
+    expires_in: int
+    token_type: str = "Bearer"  # noqa: S105
+    refresh_token: str | None = None
+
+
+class AccessTokenPayload(TokenPayload):
+    sub: str
+    vo: str
+    iss: str
+    dirac_properties: list[str]
+    preferred_username: str
+    dirac_group: str
+
+
+class RefreshTokenPayload(TokenPayload):
+    legacy_exchange: bool
+
+
+class SupportInfo(TypedDict):
+    message: str
+    webpage: str | None
+    email: str | None
+
+
+class GroupInfo(TypedDict):
+    properties: list[str]
+
+
+class VOInfo(TypedDict):
+    groups: dict[str, GroupInfo]
+    support: SupportInfo
+    default_group: str
+
+
+class Metadata(TypedDict):
+    virtual_organizations: dict[str, VOInfo]