certification.yml

# ------------------------------------------------------------
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------

name: Stable Components Certification Tests

on:
  repository_dispatch:
    types: [certification-test]
  workflow_dispatch:
  schedule:
    - cron: '5 */12 * * *'
  pull_request:
    branches:
      - master
      - release-*

jobs:
  # Based on whether this is a PR or a scheduled run, we will run a different
  # subset of the certification tests. This allows all the tests not requiring
  # secrets to be executed on pull requests.
  generate-matrix:
    runs-on: ubuntu-latest
    steps:
    - name: Install yq
      run: |
        sudo snap install yq

    - name: Specify components that can be run on every PR
      id: pr-components
      run: |
        PR_COMPONENTS=$(yq -I0 --tojson eval - << EOF
        - pubsub.kafka
        - pubsub.rabbitmq
        - pubsub.mqtt
        - state.mongodb
        - state.redis
        - state.postgresql
        - state.cassandra
        - state.memcached
        - bindings.alicloud.dubbo
        - bindings.kafka
        - bindings.redis
        - secretstores.local.env
        - secretstores.local.file
        - secretstores.hashicorp.vault
        - bindings.rabbitmq
        - bindings.localstorage
        - bindings.postgres
        EOF
        )
        echo "::set-output name=pr-components::$PR_COMPONENTS"

    - name: Specify components requiring cloud resources to run
      id: cloud-components
      run: |
        # Skip cloud-components on PRs, requires scheduled run trigger
        # or approver to trigger via respository-dispatch on /ok-to-test
        if [ "${{ github.event_name }}" = "pull_request" ]; then
          echo "::set-output name=cloud-components::[]"
          exit
        fi
        # Reuse the same cloud infrastructure as conformance.yml
        #
        # Unfortunately, Azure secrets can't have underscores in
        # names, while environment variables with hyphens ('-') are
        # troublesome.
        #
        # We work around here by leveraging the fact that
        # environment variable names are case sensitive, so
        # CamelCase would still work.
        #
        # That is slightly better than something like
        # AZURECOSMOSDBMASTERKEY, which is extremely hard to read
        # and errorprone.
        #
        # Only list the secrets you need for the component.
        CRON_COMPONENTS=$(yq -I0 --tojson eval - << EOF
        - component: secretstores.azure.keyvault
          required-secrets: AzureKeyVaultName,AzureKeyVaultSecretStoreTenantId,AzureKeyVaultSecretStoreClientId,AzureKeyVaultSecretStoreServicePrincipalClientId,AzureKeyVaultSecretStoreServicePrincipalClientSecret,AzureContainerRegistryName,AzureResourceGroupName
          required-certs: AzureKeyVaultSecretStoreCert
        - component: state.sqlserver
          required-secrets: AzureSqlServerConnectionString
        - component: bindings.azure.servicebusqueues
          required-secrets: AzureServiceBusConnectionString
        - component: bindings.azure.cosmosdb
          required-secrets: AzureCosmosDBUrl,AzureCosmosDB,AzureCosmosDBCollection,AzureCosmosDBMasterKey,AzureCertificationTenantId,AzureCertificationServicePrincipalClientId,AzureCertificationServicePrincipalClientSecret
        - component: bindings.azure.eventhubs
          required-secrets: AzureEventHubsBindingsConnectionString,AzureBlobStorageAccount,AzureBlobStorageAccessKey,AzureEventHubsBindingsHub,AzureEventHubsBindingsNamespace,AzureEventHubsBindingsConsumerGroup,AzureCertificationServicePrincipalClientId,AzureCertificationTenantId,AzureCertificationServicePrincipalClientSecret,AzureResourceGroupName,AzureCertificationSubscriptionId,AzureEventHubsBindingsContainer,AzureIotHubEventHubConnectionString,AzureIotHubName,AzureIotHubBindingsConsumerGroup
        - component: pubsub.azure.eventhubs
          required-secrets: AzureEventHubsPubsubTopicActiveConnectionString,AzureEventHubsPubsubNamespace,AzureEventHubsPubsubNamespaceConnectionString,AzureBlobStorageAccount,AzureBlobStorageAccessKey,AzureEventHubsPubsubContainer,AzureIotHubName,AzureIotHubEventHubConnectionString,AzureCertificationTenantId,AzureCertificationServicePrincipalClientId,AzureCertificationServicePrincipalClientSecret,AzureResourceGroupName,AzureCertificationSubscriptionId
        - component: pubsub.azure.servicebus
          required-secrets: AzureServiceBusConnectionString,AzureServiceBusNamespace, AzureCertificationTenantId,AzureCertificationServicePrincipalClientId,AzureCertificationServicePrincipalClientSecret
        - component: bindings.azure.blobstorage
          required-secrets: AzureBlobStorageAccount,AzureBlobStorageAccessKey,AzureBlobStorageContainer,AzureCertificationTenantId,AzureCertificationServicePrincipalClientId,AzureCertificationServicePrincipalClientSecret
        - component: bindings.azure.storagequeues
          required-secrets: AzureBlobStorageAccount, AzureBlobStorageAccessKey
        - component: state.azure.tablestorage
          required-secrets: AzureBlobStorageAccount, AzureBlobStorageAccessKey, AzureCertificationTenantId, AzureCertificationServicePrincipalClientId, AzureCertificationServicePrincipalClientSecret
        - component: state.azure.blobstorage
          required-secrets: AzureBlobStorageContainer,AzureBlobStorageAccount, AzureBlobStorageAccessKey, AzureCertificationTenantId, AzureCertificationServicePrincipalClientId, AzureCertificationServicePrincipalClientSecret
        - component: state.azure.cosmosdb
          required-secrets: AzureCosmosDBMasterKey, AzureCosmosDBUrl, AzureCosmosDB, AzureCosmosDBCollection, AzureCertificationTenantId, AzureCertificationServicePrincipalClientId, AzureCertificationServicePrincipalClientSecret    
        EOF
        )
        echo "::set-output name=cloud-components::$CRON_COMPONENTS"
    outputs:
      pr-components: ${{ steps.pr-components.outputs.pr-components }}
      cloud-components: ${{ steps.cloud-components.outputs.cloud-components }}

  certification:
    name: ${{ matrix.component }} certification
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    needs: generate-matrix
    env:
      PROJECT_PATH: ./src/github.com/dapr/components-contrib

    strategy:
      fail-fast: false # Keep running even if one component fails
      matrix:
        component: ${{ fromJson(needs.generate-matrix.outputs.pr-components) }}
        include: ${{ fromJson(needs.generate-matrix.outputs.cloud-components) }}

    steps:
    - name: Set default payload repo and ref
      run: |
        echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
        echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV

    - name: Parse repository_dispatch payload
      if: github.event_name == 'repository_dispatch'
      run: |
        if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
          echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
        fi

    - name: Check out code onto GOPATH
      uses: actions/checkout@v2
      with:
        path: ${{ env.PROJECT_PATH }}
        repository: ${{ env.CHECKOUT_REPO }}
        ref: ${{ env.CHECKOUT_REF }}

    - name: Setup test output
      run: |
        export TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report
        echo "TEST_OUTPUT_FILE_PREFIX=$TEST_OUTPUT_FILE_PREFIX" >> $GITHUB_ENV

    - name: Configure certification test and source path
      run: |
        TEST_COMPONENT=$(echo ${{ matrix.component }} | sed -E 's/\./\//g')
        export TEST_PATH="${PROJECT_PATH}/tests/certification/${TEST_COMPONENT}"
        echo "TEST_PATH=$TEST_PATH" >> $GITHUB_ENV
        export SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}"
        echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV

    - uses: Azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}
      if: matrix.required-secrets != ''

    - name: Setup secrets
      uses: Azure/get-keyvault-secrets@v1
      with:
        # Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal:
        #    az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID
        keyvault: ${{ secrets.AZURE_KEYVAULT }}
        secrets: ${{ matrix.required-secrets }}
      id: get-azure-secrets
      if: matrix.required-secrets != ''

    # Download the required certificates into files, and set env var pointing to their names
    - name: Setup certs
      if: matrix.required-certs != ''
      working-directory: ${{ env.TEST_PATH }}
      run: |
        for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
          CERT_FILE=$(mktemp --suffix .pfx)
          echo "Downloading cert $CERT_NAME into file $CERT_FILE"
          rm $CERT_FILE && \
            az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE
          echo 'Setting $CERT_NAME to' "$CERT_FILE"
          echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV
        done

    - name: Set up Go
      uses: actions/setup-go@v2
      with:
        go-version: '^1.19'

    - name: Download Go dependencies
      working-directory: ${{ env.TEST_PATH }}
      run: |
        go mod download
        go install gotest.tools/gotestsum@latest
        go install github.com/axw/gocov/gocov@v1.1.0

    - name: Check that go mod tidy is up-to-date
      working-directory: ${{ env.TEST_PATH }}
      run: |
        go mod tidy -compat=1.19
        git diff --exit-code ./go.mod
        git diff --exit-code ./go.sum

    - name: Run tests
      continue-on-error: false
      working-directory: ${{ env.TEST_PATH }}
      run: |
        echo "Running certification tests for ${{ matrix.component }} ... "

        set +e
        gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \
          --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \
          -coverprofile=cover.out -covermode=set -coverpkg=${{ env.SOURCE_PATH }}
        status=$?
        echo "Completed certification tests for ${{ matrix.component }} ... "
        if test $status -ne 0; then
          echo "Setting CERTIFICATION_FAILURE"
          export CERTIFICATION_FAILURE=true
        fi
        set -e

        COVERAGE_REPORT=$(gocov convert cover.out | gocov report)
        COVERAGE_LINE=$(echo $COVERAGE_REPORT  | grep -oP '(?<=Total Coverage:).*') # example: "80.00% (40/50)"
        COVERAGE_PERCENTAGE=$(echo $COVERAGE_LINE | grep -oP '([0-9\.]*)' | head -n 1) # example "80.00"
        echo "COVERAGE_LINE=$COVERAGE_LINE" >> $GITHUB_ENV
        echo "COMPONENT_PERCENTAGE=$COVERAGE_PERCENTAGE" >> $GITHUB_ENV

        # Fail the step if we found no test to run
        if grep -q "\[no test files\]" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json ; then
          echo "::error:: No certification test file was found for component ${{ matrix.component }}"
          exit -1
        fi

        for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
          CERT_FILE=$(printenv $CERT_NAME)

          echo "Cleaning up the certificate file $CERT_FILE..."
          rm $CERT_FILE
        done

        if [[ -v CERTIFICATION_FAILURE ]]; then
          exit 1
        fi

    - name: Prepare Cert Coverage Info
      if: github.event_name == 'schedule'
      run: |
       mkdir -p tmp/cov_files
       SOURCE_PATH_LINEAR=$(echo ${{ env.SOURCE_PATH }} |sed 's#/#\.#g') # converts slashes to dots in this string, so that it doesn't consider them sub-folders
       echo "${{ env.COVERAGE_LINE }}" >> tmp/cov_files/$SOURCE_PATH_LINEAR.txt

    - name: Upload Cert Coverage Artifact
      uses: actions/upload-artifact@v3
      if: github.event_name == 'schedule'
      with:
        name: certtest_cov
        path: tmp/cov_files
        retention-days: 1

    - name: Component Coverage Discord Notification
      if: github.event_name == 'schedule'
      env:
        DISCORD_WEBHOOK: ${{ secrets.DISCORD_MONITORING_WEBHOOK_URL }}
      uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9
      continue-on-error: true
      with:
        args: 'Cert Test Coverage for {{ SOURCE_PATH }} is {{ COVERAGE_LINE }}'

    # Upload logs for test analytics to consume
    - name: Upload test results
      if: always()
      uses: actions/upload-artifact@master
      with:
        name: ${{ matrix.component }}_certification_test
        path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.*

  post_job:
    name: Notify Total coverage
    runs-on: ubuntu-latest
    needs: certification
    if: github.event_name == 'schedule'
    steps:
      - name: Download Cert Coverage Artifact
        uses: actions/download-artifact@v3
        continue-on-error: true
        id: download
        with:
          name: certtest_cov
          path: tmp/cov_files

      - name: Calculate total coverage
        run: |
          threshold=80.0
          echo "threshold=$threshold" >> $GITHUB_ENV
          aboveThreshold=0
          totalFiles=0
          ls "${{steps.download.outputs.download-path}}" | while read f; do
          while read LINE;
              do
              ratio=$(echo $LINE | cut -d "(" -f2 | cut -d ")" -f1)
              prcnt=$(echo $LINE | cut -d "(" -f1 | cut -d ")" -f1)
              tempPrcnt=$(echo $prcnt | cut -d'%' -f1)
              if [ $tempPrcnt \> $threshold ]; then aboveThreshold=$(($aboveThreshold+1)); fi
              totalFiles=$(($totalFiles+1))
              tempNumerator=$(echo $ratio | cut -d'/' -f1)
              tempDenominator=$(echo $ratio | cut -d'/' -f2)
              export numerator=$(($numerator+$tempNumerator))
              export denominator=$(($denominator+$tempDenominator))
              totalPer=$(awk "BEGIN { print (($numerator / $denominator) * 100) }")
              echo "totalPer=$totalPer" >> $GITHUB_ENV
              echo "aboveThreshold=$aboveThreshold" >> $GITHUB_ENV
              echo "totalFiles=$totalFiles" >> $GITHUB_ENV
            done < ${{steps.download.outputs.download-path}}/$f
          done
        continue-on-error: true
      
      - name: Final Coverage Discord Notification
        env:
          DISCORD_WEBHOOK: ${{ secrets.DISCORD_MONITORING_WEBHOOK_URL }}
        uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9
        continue-on-error: true
        with:
          args: 'Total Coverage for Certification Tests is {{ totalPer }}%. {{ aboveThreshold }} out of {{ totalFiles }} components have certification tests with code coverage > {{ threshold }}%'