Merge branch 'PMM-2.0' into Kubernetes_DB_Cluster-update

Percona-Lab · Feb 21, 2024 · 643fa83 · 643fa83
2 parents e0fe7c6 + ca0d3a9
commit 643fa83
Show file tree

Hide file tree

Showing 21 changed files with 124 additions and 61 deletions.
diff --git a/.github/workflows/pmm2-testsuite.yml b/.github/workflows/pmm2-testsuite.yml
@@ -71,12 +71,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        db-type: [ps5.7, ps8, ms8.0, pdpgsql13, pdpgsql14, pdpgsql15, modb4.4, modb5, modb6, help, generic, clientContainer, haproxy, proxysql, remove]
+        db-type: [ps5.7, ps8, ms8.0, pdpgsql13, pdpgsql14, pdpgsql15, modb4.4, modb5, modb6, help, generic, server-container, clientContainer, haproxy, proxysql, remove]
 
     steps:
       - name: Create status check
         if: ${{ env.SHA != 'null' }}
-        uses: percona-platform/github-status-action@update-node
+        uses: percona/gh-action-github-status-action@update-node
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}
@@ -104,12 +104,10 @@ jobs:
       - name: Install playwright
         working-directory: ./pmm-ui-tests/cli
         run: |
-          npm install
-          npx playwright install
+          npm ci
 
       - name: Setup tools
         run: |
-          npm install -g bats
           sudo apt-get install -y apt-transport-https ca-certificates dirmngr ansible libaio1 libaio-dev libnuma-dev libncurses5 socat sysbench
           sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
           echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
@@ -123,11 +121,11 @@ jobs:
       - name: Setup PMM2-Server
         run: |
           docker create -v /srv --name pmm-server-data ${{ env.SERVER_IMAGE }}
-          docker run -d -p 80:80 -p 443:443 -p 9000:9000 -e PERCONA_TEST_PLATFORM_ADDRESS=https://check-dev.percona.com:443 -e PERCONA_TEST_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX --volumes-from pmm-server-data --name pmm-server --restart always ${{ env.SERVER_IMAGE }}
-          sleep 30
-
-      - name: healthcheck PMM2-server
-        run: timeout 100 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost/ping)" != "200" ]]; do sleep 5; done' || false
+          docker run -d -p 80:80 -p 443:443 -p 9000:9000 \
+            -e PERCONA_TEST_PLATFORM_ADDRESS=https://check-dev.percona.com:443 \
+            -e PERCONA_TEST_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX \
+            --volumes-from pmm-server-data --name pmm-server --restart always ${{ env.SERVER_IMAGE }}
+          timeout 240 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://127.0.0.1/v1/readyz)" != "200" ]]; do sleep 2; done' || false
 
       - name: Setup PMM2-Client
         run: sudo -E bash -x ./pmm-tests/pmm2-client-setup.sh --pmm_server_ip 127.0.0.1 --client_version ${{ env.CLIENT_VERSION }} --admin_password admin --use_metrics_mode no
@@ -139,12 +137,12 @@ jobs:
       - name: Run CLI '--help' tests
         if: ${{ matrix.db-type == 'help' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test pmm-ui-tests/cli/tests/help.spec.ts
+        run: npx playwright test pmm-ui-tests/cli/tests/help.spec.ts --quiet
 
       - name: Run CLI remove tests
         if: ${{ matrix.db-type == 'remove' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test pmm-ui-tests/cli/tests/remove.spec.ts
+        run: npx playwright test pmm-ui-tests/cli/tests/remove.spec.ts --quiet
 
       - name: Run Setup for ps5.7
         working-directory: pmm-integration
@@ -158,10 +156,10 @@ jobs:
         run: |
           sudo npx ts-node ./integration-setup.ts --ps-version=8 --ci --addclient=ps,1
 
-      - name: Run bats tests for PS
+      - name: Run CLI tests for PS
         if: ${{ matrix.db-type == 'ps5.7' || matrix.db-type == 'ps8' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test pmm-ui-tests/cli/tests/perconaMySqlServer.spec.ts
+        run: npx playwright test pmm-ui-tests/cli/tests/perconaMySqlServer.spec.ts --quiet
 
       - name: Run Setup for pdpgsql 13
         working-directory: pmm-integration
@@ -184,27 +182,34 @@ jobs:
       - name: Run CLI tests for PDPGSQL
         if: ${{ matrix.db-type == 'pdpgsql13' || matrix.db-type == 'pdpgsql14' || matrix.db-type == 'pdpgsql15' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test postgreSql
+        run: npx playwright test postgreSql --quiet
 
       - name: Run Setup for PSMDB 4.4
         working-directory: pmm-integration
         if: ${{ matrix.db-type == 'modb4.4' }}
-        run: sudo npx ts-node ./integration-setup.ts --ci --addclient=modb,1 --mo-version=4.4
+        run: |
+          sudo npx ts-node ./integration-setup.ts --ci --addclient=modb,1 --mo-version=4
+          sudo -E bash -x ../pmm-tests/pmm-framework.sh --addclient=modb,1 --mo-version=4 --with-shard --mongomagic    
 
       - name: Run Setup for PSMDB 5
         working-directory: pmm-integration
         if: ${{ matrix.db-type == 'modb5' }}
-        run: sudo npx ts-node ./integration-setup.ts --ci --addclient=modb,1 --mo-version=5
+        run: |
+          sudo npx ts-node ./integration-setup.ts --ci --addclient=modb,1 --mo-version=5
+          sudo -E bash -x ../pmm-tests/pmm-framework.sh --addclient=modb,1 --mo-version=5 --with-shard --mongomagic
 
       - name: Run Setup for PSMDB 6
         working-directory: pmm-integration
         if: ${{ matrix.db-type == 'modb6' }}
-        run: sudo npx ts-node ./integration-setup.ts --ci --addclient=modb,1 --mo-version=6
+        run: |
+          sudo npx ts-node ./integration-setup.ts --ci --addclient=modb,1 --mo-version=6
+          sudo -E bash -x ../pmm-tests/pmm-framework.sh --addclient=modb,1 --mo-version=6 --with-shard --mongomagic
 
       - name: Run CLI tests for PSMDB
         if: ${{ matrix.db-type == 'modb6' || matrix.db-type == 'modb5' || matrix.db-type == 'modb4.4' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test pmm-ui-tests/cli/tests/mongoDb-psmdb.spec.ts
+        run: |
+          npx playwright test pmm-ui-tests/cli/tests/mongoDb-psmdb.spec.ts --quiet
 
 # TODO: Add setup for the official MongoDB and run "npx playwright test pmm-ui-tests/cli/tests/mongoDb.spec.ts"
 
@@ -216,17 +221,17 @@ jobs:
       - name: Run CLI tests for Generic Scenarios
         if: ${{ matrix.db-type == 'generic' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test generic unregister
+        run: npx playwright test generic unregister --quiet
 
-      - name: Run bats tests for generic Scenarios
-        if: ${{ matrix.db-type == 'generic' }}
-        run: |
-          sudo bats ./pmm-tests/pmm-2-0-bats-tests/generic-tests.bats
+      - name: Run CLI tests for PMM Server Container
+        if: ${{ matrix.db-type == 'server-container' && !contains(env.PMM_UI_BRANCH, 'pmm-2.') }}
+        working-directory: ./pmm-ui-tests/cli
+        run: npx playwright test pmm-server-only --quiet
 
       - name: Run CLI tests for PMM Client Docker Container
         if: ${{ matrix.db-type == 'clientContainer' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test pmm-client-docker
+        run: npx playwright test pmm-client-docker --quiet
 
       - name: Run Setup for HAPROXY
         if: ${{ matrix.db-type == 'haproxy' }}
@@ -238,7 +243,7 @@ jobs:
       - name: Run CLI tests for Client Container Scenarios
         if: ${{ matrix.db-type == 'haproxy' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test haproxy
+        run: npx playwright test haproxy --quiet
 
       - name: Run Setup for ProxySQL
         if: ${{ matrix.db-type == 'proxysql' }}
@@ -247,7 +252,7 @@ jobs:
       - name: Run CLI tests for ProxySQL
         if: ${{ matrix.db-type == 'proxysql' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test proxySql
+        run: npx playwright test proxySql --quiet
 
       - name: Run Setup for MySQL 8.0
         if: ${{ matrix.db-type == 'ms8.0' }}
@@ -256,23 +261,23 @@ jobs:
       - name: Run MySQL specific CLI tests
         if: ${{ matrix.db-type == 'ms8.0' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test pmm-ui-tests/cli/tests/mysql.spec.ts
+        run: npx playwright test pmm-ui-tests/cli/tests/mysql.spec.ts --quiet
 
       - name: Run CLI tests for MySQL Tests
         if: ${{ matrix.db-type == 'ms8.0' }}
         working-directory: ./pmm-ui-tests/cli
-        run: npx playwright test pmm-ui-tests/cli/tests/mysql-conf-file.spec.ts
+        run: npx playwright test pmm-ui-tests/cli/tests/mysql-conf-file.spec.ts --quiet
 
       - name: Generate and Attach the report
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: "report-${{ matrix.db-type }}"
           path: ./pmm-ui-tests/cli/test-report
 
       - name: Create status check
         if: ${{ always() }}
-        uses: percona-platform/github-status-action@update-node
+        uses: percona/gh-action-github-status-action@update-node
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/pmm2-ui-tests-fb.yml b/.github/workflows/pmm2-ui-tests-fb.yml
@@ -106,6 +106,20 @@ jobs:
       #      tags_for_tests: '@bm-common|@bm-locations' -- include bm-common once setup for mysql fixed
       tags_for_tests: '@bm-locations'
 
+  exporters:
+    name: Exporters tests
+    uses: ./.github/workflows/ui-tests.yml
+    secrets: inherit
+    with:
+      server_image: ${{ inputs.server_image || 'perconalab/pmm-server:dev-latest' }}
+      client_version: ${{ inputs.client_version || 'dev-latest' }}
+      client_image: ${{ inputs.client_image || 'perconalab/pmm-client:dev-latest' }}
+      pmm_qa_branch: ${{ inputs.pmm_qa_branch || 'main' }}
+      pmm_ui_branch: ${{ inputs.pmm_ui_branch || 'main' }}
+      sha: ${{ inputs.sha || github.event.pull_request.head.sha ||  'null' }}
+      client_flags: '--mongo-replica-for-backup'
+      tags_for_tests: '@mongodb-exporter|@exporters'
+
   instances:
     name: Instances UI tests
     uses: ./.github/workflows/ui-tests.yml
@@ -215,3 +229,39 @@ jobs:
       sha: ${{ inputs.sha || github.event.pull_request.head.sha ||  'null' }}
       client_flags: '--addclient=ps,1 --addclient=modb,1 --addclient=pdpgsql,1'
       tags_for_tests: '@user-password'
+
+  dump_tool:
+    name: Dump Tool UI tests
+    uses: ./.github/workflows/ui-tests.yml
+    secrets: inherit
+    with:
+      server_image: ${{ inputs.server_image || 'perconalab/pmm-server:dev-latest' }}
+      client_version: ${{ inputs.client_version || 'dev-latest' }}
+      client_image: ${{ inputs.client_image || 'perconalab/pmm-client:dev-latest' }}
+      pmm_qa_branch: ${{ inputs.pmm_qa_branch || 'main' }}
+      pmm_ui_branch: ${{ inputs.pmm_ui_branch || 'main' }}
+      sha: ${{ inputs.sha || github.event.pull_request.head.sha ||  'null' }}
+      client_flags: '--addclient=ps,1'
+      tags_for_tests: '@dump'
+
+  tarball_install:
+    name: Tarball Install test
+    uses: Percona-QA/package-testing/.github/workflows/pmm-package-test-fb.yml@master
+    secrets: inherit
+    with:
+      package_testing_branch: 'master'
+      pmm_server_image: ${{ inputs.pmm_server_image }}
+      pmm_client_tarball: ${{ inputs.client_version || 'dev-latest' }}
+      playbook: 'pmm2-client_integration_custom_port'
+      test_name: 'Tarball install'
+
+  tarball_upgrade:
+    name: Tarball Upgrade test
+    uses: Percona-QA/package-testing/.github/workflows/pmm-package-test-fb.yml@master
+    secrets: inherit
+    with:
+      package_testing_branch: 'master'
+      pmm_server_image: ${{ inputs.server_image || 'perconalab/pmm-server:dev-latest' }}
+      pmm_client_tarball: ${{ inputs.client_version || 'dev-latest' }}
+      playbook: 'pmm2-client_integration_upgrade_custom_path'
+      test_name: 'Tarball upgrade'
diff --git a/.github/workflows/trivy_scan.yml b/.github/workflows/trivy_scan.yml
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       SHA: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha }}
-    
+
     strategy:
       fail-fast: false
       matrix:
@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: Communicate action status back to the PR
-        uses: percona-platform/github-status-action@update-node
+        uses: percona/gh-action-github-status-action@update-node
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}
@@ -59,15 +59,15 @@ jobs:
 
       - name: Archive trivy scan results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.REPORT }}
           path: ${{ env.REPORT }}
           retention-days: 7
 
       - name: Communicate action status back to the PR
         if: always()
-        uses: percona-platform/github-status-action@update-node
+        uses: percona/gh-action-github-status-action@update-node
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/ui-tests-podman.yml b/.github/workflows/ui-tests-podman.yml
@@ -67,7 +67,7 @@ jobs:
     steps:
       - name: Create status check
         if: ${{ github.event_name != 'pull_request' }}
-        uses: percona-platform/github-status-action@v1
+        uses: percona/gh-action-github-status-action@v1
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}
@@ -213,7 +213,7 @@ jobs:
           sed -i 's+http://localhost/+${PMM_UI_URL}/+g' pr.codecept.js
           ./node_modules/.bin/codeceptjs run -c pr.codecept.js --grep "${{ env.TAGS_FOR_TESTS }}"
 
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         if: ${{ always() }}
         id: artifact_name
         with:
@@ -223,7 +223,7 @@ jobs:
 
       - name: Create status check
         if: ${{ github.event_name != 'pull_request' && always() }}
-        uses: percona-platform/github-status-action@v1
+        uses: percona/gh-action-github-status-action@v1
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml
@@ -45,6 +45,7 @@ jobs:
       SERVER_IMAGE: ${{ inputs.server_image || 'perconalab/pmm-server:dev-latest' }}
       CLIENT_IMAGE: ${{ inputs.client_image || 'perconalab/pmm-client:dev-latest' }}
       CLIENT_VERSION: ${{ inputs.client_version || 'dev-latest' }}
+      PMM_CLIENT_VERSION: ${{ inputs.client_version || 'dev-latest' }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       ADMIN_PASSWORD: 'admin-password'
       PMM_QA_BRANCH: ${{ inputs.pmm_qa_branch || 'main' }}
@@ -64,7 +65,7 @@ jobs:
     steps:
       - name: Create status check
         if: ${{ github.event_name != 'pull_request' }}
-        uses: percona-platform/github-status-action@v1
+        uses: percona/gh-action-github-status-action@v1
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}
@@ -136,17 +137,24 @@ jobs:
           sed -i 's+http://localhost/+${PMM_UI_URL}/+g' pr.codecept.js
           ./node_modules/.bin/codeceptjs run -c pr.codecept.js --grep "${{ env.TAGS_FOR_TESTS }}"
 
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         if: ${{ always() }}
         id: artifact_name
         with:
           script: |
             return `artifacts_for_${process.env.TAGS_FOR_TESTS.replaceAll('|', '')}`
           result-encoding: string
 
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: ${{ steps.artifact_name.outputs.result }}
+          path: |
+            pmm-ui-tests/tests/output/*.png
+
       - name: Create status check
         if: ${{ github.event_name != 'pull_request' && always() }}
-        uses: percona-platform/github-status-action@v1
+        uses: percona/gh-action-github-status-action@v1
         continue-on-error: true
         with:
           authToken: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.gitmodules b/.gitmodules
@@ -16,7 +16,7 @@
 [submodule "mongodb_exporter"]
 	path = sources/mongodb_exporter/src/github.com/percona/mongodb_exporter
 	url = https://github.com/percona/mongodb_exporter.git
-	branch = release-0.39.0
+	branch = main
 [submodule "postgres_exporter"]
 	path = sources/postgres_exporter/src/github.com/percona/postgres_exporter
 	url = https://github.com/percona/postgres_exporter
@@ -40,7 +40,7 @@
 [submodule "percona-toolkit"]
 	path = sources/percona-toolkit/src/github.com/percona/percona-toolkit
 	url = https://github.com/percona/percona-toolkit.git
-	branch = release-v3.5.2
+	branch = release-v3.5.5
 
 # PMM Server
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.41.0
+2.41.2
diff --git a/ci-default.yml b/ci-default.yml
@@ -20,7 +20,7 @@ deps:
     component: client
 
   - name: mongodb_exporter
-    branch: release-0.39.0
+    branch: main
     path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter
     url: https://github.com/percona/mongodb_exporter
     component: client
@@ -56,7 +56,7 @@ deps:
     component: client
 
   - name: percona-toolkit
-    branch: release-v3.5.2
+    branch: release-v3.5.5
     path: sources/percona-toolkit/src/github.com/percona/percona-toolkit
     url: https://github.com/percona/percona-toolkit
     component: client

diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
-pyaml==23.9.5
-GitPython==3.1.37
-PyGithub==1.59.1
+pyaml==23.12.0
+GitPython==3.1.42
+PyGithub==2.2.0