Merge pull request #95 from ricardogsilva/94-add-workflow-for-integration-tests

Add a workflow for testing github action

Author: ricardogsilva

.github/workflows/test-action.yaml

@@ -0,0 +1,57 @@
+name: Test cite-runner as a GitHub action
+
+on:
+  # will run both when pushing a branch and when pushing a tag
+  push:
+
+  pull_request:
+
+  # will run when called from another workflow
+  workflow_call:
+
+env:
+  COLUMNS: 120
+
+jobs:
+  test_cite_runner_github_action:
+    runs-on: ubuntu-24.04
+    env:
+      SIMPLESERVER_PORT: 9092
+    steps:
+      - name: "Grab code"
+        uses: actions/checkout@v4.2.2
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+      - name: "Launch simple test server"
+        run: |
+          nohup python tests/simpleserver.py --bind-address 0.0.0.0 --bind-port ${{ env.SIMPLESERVER_PORT }} > /dev/null 2>&1 &
+          echo $! > simpleserver.pid
+          sleep 1
+          if ps --pid $(cat simpleserver.pid) > /dev/null; then
+              echo "simpleserver is up and running with PID $(cat simpleserver.pid)"
+          else
+              echo "simpleserver failed to start"
+              exit 1
+          fi
+      - name: "Test cite-runner GitHub action"
+        id: test_cite_runner_github_action
+        uses: ./
+        with:
+          test_suite_identifier: ogcapi-features-1.0
+          test_session_arguments: iut=http://host.docker.internal:${{ env.SIMPLESERVER_PORT }}
+          exit_with_error: "false"
+      - name: "Verify cite-runner results"
+        run: |
+          jq '.' <<EOF | python tests/github_action_tests.py -
+          ${{ steps.test_cite_runner_github_action.outputs.json_report }}
+          EOF
+      - name: "Shut down simple test server"
+        if: always()
+        run: |
+          if [ -f server.pid ]; then
+              echo "Shutting down simpleserver with PID $(cat simpleserver.pid)..."
+              kill $(cat simpleserver.pid) || true
+              rm simpleserver.pid
+          fi

action.yml

@@ -50,29 +50,36 @@ inputs:
     description: Whether the action exits with an error if the test suite execution shows a result of failed.
 
 outputs:
-  results:
-    description: "Test results"
-    value: '${{ steps.run_cite_runner.outputs.MARKDOWN_RESULT_OUTPUT_PATH }}'
+  json_report:
+    description: "Full test suite execution result as JSON"
+    value: '${{ steps.run_cite_runner.outputs.JSON_REPORT }}'
 
 runs:
   using: 'composite'
   steps:
     - name: "Add action path to the global path"
       shell: bash
       run: echo "${{ github.action_path }}" >> ${GITHUB_PATH}
+    - name: "normalize action path"
+      id: normalize_action_path
+      shell: "bash"
+      run: |
+        echo "ACTION_NORMALIZED_PATH=$(realpath -m ${{ github.action_path }})" >> "${GITHUB_OUTPUT}"
     - name: "Install uv"
       uses: astral-sh/setup-uv@v5
       with:
         version: "0.6.13"
         enable-cache: true
-        cache-dependency-glob: "${{ github.action_path }}/uv.lock"
+        cache-dependency-glob: "${{ steps.normalize_action_path.outputs.ACTION_NORMALIZED_PATH }}/uv.lock"
     - name: "Set up Python"
       uses: actions/setup-python@v5
       with:
         python-version-file: "${{ github.action_path }}/pyproject.toml"
     - name: "Install cite-runner"
       env:
-        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CITE_RUNNER: ${{ github.action_ref }}
+#        GITHUB_CONTEXT: ${{ toJSON(github) }}
+#        ENV_CONTEXT: ${{ toJSON(env) }}
+        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CITE_RUNNER: ${{ env.GITHUB_ACTION_REF }}
       shell: "bash"
       run: |
         cd ${{ github.action_path }}
@@ -85,7 +92,8 @@ runs:
         --detach
         --rm
         --name teamengine
-        --network host
+        --add-host=host.docker.internal:host-gateway
+        --publish 9080:8080
         ogccite/teamengine-production:1.0-SNAPSHOT
     - name: "Run cite-runner"
       id: "run_cite_runner"
@@ -96,13 +104,17 @@ runs:
         uv run cite-runner \
             --network-timeout=${{ inputs.network_timeout_seconds }} \
             execute-test-suite-from-github-actions \
-            ${{ inputs.teamengine_url || 'http://localhost:8080/teamengine' }} \
+            ${{ inputs.teamengine_url || 'http://localhost:9080/teamengine' }} \
             ${{ inputs.test_suite_identifier }} \
             --output-format=raw \
             --teamengine-username=${{ inputs.teamengine_username }} \
             --teamengine-password=${{ inputs.teamengine_password }} \
             $(echo -e ${{ inputs.test_session_arguments }}) 1> ${RAW_PATH}
-        echo "RAW_RESULT_OUTPUT_PATH=${{ github.action_path }}/${RAW_PATH}" >> "${GITHUB_OUTPUT}"
+        if [ $? -ne 0 ]; then
+            echo "Unable to use TeamEngine to execute test suite ${{ inputs.test_suite_identifier }} with arguments ${{ inputs.test_session_arguments }}"
+            exit 1
+        fi
+        echo "RAW_RESULT_OUTPUT_PATH=${{ steps.normalize_action_path.outputs.ACTION_NORMALIZED_PATH }}/${RAW_PATH}" >> "${GITHUB_OUTPUT}"
         echo "::group::cite-runner suite execution results"
         uv run cite-runner parse-result \
             --output-format console \
@@ -123,17 +135,20 @@ runs:
             ${{ fromJSON(inputs.with_passed) && '--with-passed \' || '--without-passed \'}}
             ${RAW_PATH} 1> ${MARKDOWN_PATH}
         cat ${MARKDOWN_PATH} >> ${GITHUB_STEP_SUMMARY}
-        echo "MARKDOWN_RESULT_OUTPUT_PATH=${{ github.action_path }}/${MARKDOWN_PATH}" >> "${GITHUB_OUTPUT}"
+        echo "MARKDOWN_RESULT_OUTPUT_PATH=${{ steps.normalize_action_path.outputs.ACTION_NORMALIZED_PATH }}/${MARKDOWN_PATH}" >> "${GITHUB_OUTPUT}"
+        echo "JSON_REPORT=$(uv run cite-runner parse-result \
+            --output-format json \
+            ${RAW_PATH}
+        )" >> "${GITHUB_OUTPUT}"
     - name: "Store execution results as artifacts"
-      if: ${{ !cancelled() }}
       uses: actions/upload-artifact@v4
       with:
         name: 'execution-results-${{ inputs.test_suite_identifier }}'
         path: |
           ${{ steps.run_cite_runner.outputs.RAW_RESULT_OUTPUT_PATH }}
           ${{ steps.run_cite_runner.outputs.MARKDOWN_RESULT_OUTPUT_PATH }}
-    - name: "Stop TEAM engine container"
-      if: ${{ !cancelled() && !inputs.teamengine_url }}
+    - name: "Stop teamengine container"
+      if: ${{ always() && !inputs.teamengine_url }}
       shell: "bash"
       run: docker stop teamengine
     - name: "Set action exit code"

docs/development.md

@@ -57,31 +57,78 @@ In a brief nutshell:
     docker run \
         --rm \
         --name=teamengine \
-        --network=host \
+        --add-host=host.docker.internal:host-gateway \
+        --publish=9080:8080 \
         ogccite/teamengine-production:1.0-SNAPSHOT
     ```
 
-    You should now be able to use `http:localhost:8080/teamengine` in
-    cite-runner
+    You should now be able to use `http:localhost:9080/teamengine` as the teamengine URL in
+    cite-runner.
 
-    !!! warning
-
-        teamengine will try to run on your local port `8080`, which could
-        potentially already be occupied by another application.
+    !!! note
 
-7. Work on the cite-runner code
+        Using docker's `--add-host=host.docker.internal:host-gateway` is necessary when running
+        docker engine, as discussed in the [docker engine docs:material-open-in-new:]{: target="blank_" }. If
+        you are using docker desktop you can omit this flag.
 
-8. You can run cite-runner via uv with:
+7.  You can run cite-runner via uv with:
 
     ```shell
     uv run cite-runner
     ```
 
-8. If you want to work on documentation, you can start the mkdocs server with:
+    !!! warning
 
-    ```shell
-    uv run mkdocs serve
-    ```
+         When using cite-runner with a local teamengine instance that is running via docker and also testing an
+         OGC service that is running locally on the same machine, you must not use `localhost` when providing the
+         service's URL to teamengine, but rather use `host.docker.internal`.
+
+         As an example:
+
+         ```shell
+         uv run cite-runner execute-test-suite \
+             http://localhost:9081/teamengine \
+             ogcapi-features-1.0 \
+             --suite-input iut http://host.docker.internal:9082
+         ```
+
+
+### Running tests
+
+Most tests can be run with:
+
+ ```shell
+ uv run pytest
+ ```
+
+cite-runner also includes a workflow for testing itself when running as a GitHub action. This can be run locally
+with a tool like [act:material-open-in-new:]{: target="blank_" }.
+
+ ```shell
+ act \
+     --workflows .github/workflows/test-action.yaml \
+     --rm \
+     --platform ubuntu-24.04=ghcr.io/catthehacker/ubuntu:act-24.04 \
+     --container-options="-p 9092:9092" \
+     --artifact-server-path $PWD/.artifacts
+ ```
+
+The `.github/workflows/test-action.yaml` workflow launches a simple HTTP server which contains a very incomplete
+implementation of OGC API - Features and then uses the cite-runner GitHub action to run the `ogcapi-features-1.0`
+test suite on it. It then captures the cite-runner output, and runs it through some Python tests to verify the
+result matches what is expected.
+
+
+### Documentation
+
+If you want to work on documentation, you can start the mkdocs server with:
+
+ ```shell
+ uv run mkdocs serve
+ ```
+
+Now edit files under the `/docs` directory and check whether they match your expected result in the mkdocs dev server,
+which would be running at `http://localhost:8000/cite-runner/
 
 
 ## Release management
@@ -100,6 +147,8 @@ set up to run whenever a new tag named `v*` is pushed to the repository. This wo
 
 
 
+[act:material-open-in-new:]: https://nektosact.com/introduction.html
+[docker engine docs:material-open-in-new:]: https://docs.docker.com/reference/cli/docker/container/run/#add-host
 [GitHub actions workflow:material-open-in-new:]: https://github.com/OSGeo/cite-runner/blob/main/.github/workflows/release.yaml
 [httpx:material-open-in-new:]: https://www.python-httpx.org/
 [jinja:material-open-in-new:]: https://jinja.palletsprojects.com/en/stable/

docs/running-as-github-action.md

@@ -18,9 +18,6 @@ and specify which test suite to run, alongside any relevant parameters.
     Although cite-runner is not yet published in the [GitHub marketplace:material-open-in-new:]{: target="blank_" } it
     can still be used in GitHub CI workflows.
 
-[GitHub action:material-open-in-new:]: https://docs.github.com/en/actions/sharing-automations/creating-actions/about-custom-actions
-[GitHub marketplace:material-open-in-new:]: https://github.com/marketplace
-
 Include it as any other GitHub action, by creating a workflow step that
 specifies `uses: OSGEO/cite-runner` and provide execution parameters in the
 `with` parameter.
@@ -202,10 +199,35 @@ When run as a GitHub action, cite-runner expects the following inputs to be prov
 
 
 
-[GitHub actions inputs:material-open-in-new:]: https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#inputs
-[fromJSON() function:material-open-in-new:]: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#fromjson
-[ternary operator:material-open-in-new:]: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#operators
+## Outputs
 
+The cite-runner GitHub Action will provide a single output, which is a full report of the test suite results:
+
+
+### json_report
+
+This is a JSON document containing the full parsed test suite execution results. You can use it in further GitHub
+workflow steps to verify suite execution. As an example of further processing, you can pipe the result to
+[jq:material-open-in-new:]{: target="blank_" }, as seen below:
+
+```yaml
+- name: "Verify cite-runner results"
+  run: |
+  jq '.passed' <<EOF
+  ${{ steps.test_cite_runner_github_action.outputs.json_report }}
+  EOF
+```
+
+!!! tip
+
+    Handling outputs of a GitHub Action that represent JSON data can be a bit tricky. The previous example showcases
+    using a [HERE doc:material-open-in-new:]{: target="blank_" }, which has the benefit of preserving whatever
+    double/single quotes may be present in the underlying JSON data. We recommend always using this technique to
+    process cite-runner GitHub Action's output
+
+
+[cite-runner's own testing workflow:material-open-in-new:]{: target="blank_" } has an additional example of using
+the action's output and passing it to another command for further processing.
 
 ## Usage examples
 
@@ -422,6 +444,15 @@ relevant steps consist of calling cite-runner as a standalone CLI application. B
 7. Finally, set the action exit code. This is done by retrieving the exit code that had been stored in 4.2 and using it
    to set the overall action exit code.
 
+
+[cite-runner's own testing workflow:material-open-in-new:]: https://github.com/OSGeo/cite-runner/tree/main/.github/workflows/test-action.yaml
 [composite action:material-open-in-new:]: https://docs.github.com/en/actions/sharing-automations/creating-actions/creating-a-composite-action
 [custom shell:material-open-in-new:]: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
+[jq:material-open-in-new:]: https://jqlang.org/
+[fromJSON() function:material-open-in-new:]: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#fromjson
+[GitHub action:material-open-in-new:]: https://docs.github.com/en/actions/sharing-automations/creating-actions/about-custom-actions
+[GitHub actions inputs:material-open-in-new:]: https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#inputs
+[GitHub marketplace:material-open-in-new:]: https://github.com/marketplace
+[HERE doc:material-open-in-new:]: https://linuxize.com/post/bash-heredoc/
 [set -e:material-open-in-new:]: https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#The-Set-Builtin
+[ternary operator:material-open-in-new:]: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#operators