Fixes current branch check problem when tag is pushed to push docker tags

.github/workflows/tool-tests.yml

@@ -29,6 +29,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Define git credentials
         run: git config --global user.email "${MICROSOFT_EMAIL}"&& git config --global user.name "${USER_NAME}"
@@ -42,6 +44,9 @@ jobs:
       - name: Run static code analysis
         run: python -m prospector
 
+      - name: Unit tests for "Common tools"
+        run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py
+
       - name: Unit tests for "Update Package Properties"
         run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py
 
@@ -54,8 +59,5 @@ jobs:
       - name: Unit tests for "Update Pgxn"
         run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py
 
-      - name: Unit tests for "Common tools"
-        run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py
-
       - name: Packaging Warning Handler
         run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py

packaging_automation/common_tool_methods.py

@@ -1,18 +1,18 @@
-import base64
 import os
 import re
 import shlex
 import subprocess
 from datetime import datetime
 from enum import Enum
-from typing import Dict, List
-from typing import Tuple
+from typing import Dict, List, Tuple
 
+import base64
+import git
 import gnupg
 import pathlib2
 import requests
-from git import Repo
-from github import Repository, PullRequest, Commit, Github
+from git import GitCommandError, Repo
+from github import Commit, Github, PullRequest, Repository
 from jinja2 import Environment, FileSystemLoader
 from parameters_validation import validate_parameters
 
@@ -300,8 +300,25 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool:
     return has_match
 
 
+def is_tag_on_branch(tag_name: str, branch_name: str):
+    g = git.Git(os.getcwd())
+    try:
+        branches_str = g.execute(["git", "branch", "--contains", f"tags/{tag_name}"])
+        branches = remove_prefix(branches_str, "*").split("\n")
+        print("Branches str:" + branches_str)
+        if len(branches) > 0:
+            for branch in branches:
+                if branch.strip() == branch_name:
+                    return True
+        return False
+    except GitCommandError as e:
+        print("Error:" + str(e))
+        return False
+
+
 def get_current_branch(working_dir: str) -> str:
     repo = get_new_repo(working_dir)
+
     return repo.active_branch.name
 
 

packaging_automation/publish_docker.py

@@ -7,7 +7,7 @@
 import pathlib2
 from parameters_validation import validate_parameters
 
-from .common_tool_methods import remove_prefix, get_current_branch
+from .common_tool_methods import remove_prefix, get_current_branch, is_tag_on_branch
 from .common_validations import is_tag
 
 BASE_PATH = pathlib2.Path(__file__).parents[1]
@@ -93,91 +93,94 @@ def get_image_tag(tag_prefix: str, docker_image_type: DockerImageType) -> str:
     return f"{tag_prefix}{tag_suffix}"
 
 
-def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, current_branch: str):
+def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, will_image_be_published: bool):
     triggering_event_info, resource_name = decode_triggering_event_info(github_ref)
     for regular_image_type in regular_images_to_be_built(docker_image_type):
         if triggering_event_info == GithubTriggerEventSource.branch_push:
-            publish_main_docker_images(regular_image_type, current_branch)
+            publish_main_docker_images(regular_image_type, will_image_be_published)
         else:
-            publish_tagged_docker_images(regular_image_type, resource_name, current_branch)
+            publish_tagged_docker_images(regular_image_type, resource_name, will_image_be_published)
 
 
-def publish_docker_image_on_schedule(docker_image_type: DockerImageType, current_branch: str, ):
+def publish_docker_image_on_schedule(docker_image_type: DockerImageType, will_image_be_published: bool):
     if docker_image_type == DockerImageType.nightly:
-        publish_nightly_docker_image(current_branch)
+        publish_nightly_docker_image(will_image_be_published)
     else:
         for regular_image_type in regular_images_to_be_built(docker_image_type):
-            publish_main_docker_images(regular_image_type, current_branch)
+            publish_main_docker_images(regular_image_type, will_image_be_published)
 
 
-def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, current_branch: str,
+def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, will_image_be_published: bool,
                                   docker_image_type: DockerImageType, tag_name: str = "") -> None:
     if manual_trigger_type_param == ManualTriggerType.main and not tag_name:
         for it in regular_images_to_be_built(docker_image_type):
-            publish_main_docker_images(it, current_branch)
+            publish_main_docker_images(it, will_image_be_published)
     elif manual_trigger_type_param == ManualTriggerType.tags and tag_name:
         for it in regular_images_to_be_built(docker_image_type):
-            publish_tagged_docker_images(it, tag_name, current_branch)
+            publish_tagged_docker_images(it, tag_name, will_image_be_published)
     elif manual_trigger_type_param == ManualTriggerType.nightly:
-        publish_nightly_docker_image(current_branch)
+        publish_nightly_docker_image(will_image_be_published)
 
 
-def publish_main_docker_images(docker_image_type: DockerImageType, current_branch: str):
+def publish_main_docker_images(docker_image_type: DockerImageType, will_image_be_published: bool):
     print(f"Building main docker image for {docker_image_type.name}...")
     docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_type.name}"
     docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'],
                                tag=docker_image_name,
                                path=".")
     print(f"Main docker image for {docker_image_type.name} built.")
-    if current_branch == DEFAULT_BRANCH_NAME:
+    if will_image_be_published:
         print(f"Publishing main docker image for {docker_image_type.name}...")
         docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_type.name)
         print(f"Publishing main docker image for {docker_image_type.name} finished")
     else:
-        print(
-            f"Since current branch {current_branch} is not equal to "
-            f"{DEFAULT_BRANCH_NAME} {docker_image_name} will not be pushed.")
+        current_branch = get_current_branch(os.getcwd())
+        if current_branch != DEFAULT_BRANCH_NAME:
+            print(
+                f"Since current branch {current_branch} is not equal to "
+                f"{DEFAULT_BRANCH_NAME} {docker_image_name} will not be pushed.")
 
 
-def publish_tagged_docker_images(docker_image_type, tag_name: str, current_branch: str):
+def publish_tagged_docker_images(docker_image_type, tag_name: str, will_image_be_published: bool):
     print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name}...")
     tag_parts = decode_tag_parts(tag_name)
     tag_version_part = ""
     docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_type.name}"
     docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'],
                                tag=docker_image_name,
                                path=".")
-    if current_branch != DEFAULT_BRANCH_NAME:
-        print(f"Since current branch {current_branch} is not equal to {DEFAULT_BRANCH_NAME} tags will not be pushed.")
     print(f"{docker_image_type.name} image built.Now starting tagging and pushing...")
     for tag_part in tag_parts:
         tag_version_part = tag_version_part + tag_part
         image_tag = get_image_tag(tag_version_part, docker_image_type)
         print(f"Tagging {docker_image_name} with the tag {image_tag}...")
         docker_api_client.tag(docker_image_name, docker_image_name, image_tag)
         print(f"Tagging {docker_image_name} with the tag {image_tag} finished.")
-
-        if current_branch == DEFAULT_BRANCH_NAME:
+        if will_image_be_published:
             print(f"Pushing {docker_image_name} with the tag {image_tag}...")
             docker_client.images.push(DOCKER_IMAGE_NAME, tag=image_tag)
             print(f"Pushing {docker_image_name} with the tag {image_tag} finished")
+        else:
+            print(f"Skipped pushing {docker_image_type} with the tag {image_tag} since will_image_be_published flag is false")
 
         tag_version_part = tag_version_part + "."
     print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name} finished.")
 
 
-def publish_nightly_docker_image(current_branch: str):
+def publish_nightly_docker_image(will_image_be_published: bool):
     print("Building nightly image...")
     docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_info_dict[DockerImageType.nightly]['docker-tag']}"
     docker_client.images.build(dockerfile=docker_image_info_dict[DockerImageType.nightly]['file-name'],
                                tag=docker_image_name,
                                path=".")
     print("Nightly image build finished.")
 
-    if current_branch == DEFAULT_BRANCH_NAME:
+    if will_image_be_published:
         print("Pushing nightly image...")
         docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_info_dict[DockerImageType.nightly]['docker-tag'])
         print("Nightly image push finished.")
+    else:
+        print("Nightly image will not be pushed since will_image_be_published flag is false")
 
 
 def validate_and_extract_general_parameters(docker_image_type_param: str, pipeline_trigger_type_param: str) -> Tuple[
@@ -216,23 +219,42 @@ def validate_and_extract_manual_exec_params(manual_trigger_type_param: str, tag_
     return manual_trigger_type_param
 
 
+def get_image_publish_status(github_ref: str, is_test: bool):
+    if is_test:
+        return False
+    triggering_event_info, resource_name = decode_triggering_event_info(github_ref)
+    if triggering_event_info == GithubTriggerEventSource.tag_push:
+        if not is_tag_on_branch(tag_name=resource_name, branch_name=DEFAULT_BRANCH_NAME):
+            return False
+        return True
+    current_branch = get_current_branch(os.getcwd())
+    if current_branch != DEFAULT_BRANCH_NAME:
+        return False
+    return True
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('--github_ref')
+    parser.add_argument('--github_ref', required=True)
     parser.add_argument('--pipeline_trigger_type', choices=[e.name for e in GithubPipelineTriggerType], required=True)
     parser.add_argument('--tag_name', nargs='?', default="")
     parser.add_argument('--manual_trigger_type', choices=[e.name for e in ManualTriggerType])
     parser.add_argument('--image_type', choices=[e.name for e in DockerImageType])
+    parser.add_argument('--is_test', action="store_true")
     args = parser.parse_args()
 
     pipeline_trigger_type, image_type = validate_and_extract_general_parameters(args.image_type,
                                                                                 args.pipeline_trigger_type)
-    current_branch = get_current_branch(os.getcwd())
+    if args.is_test:
+        print("Script is working in test mode. Images will not be published")
+
+    publish_status = get_image_publish_status(args.github_ref, args.is_test)
     if pipeline_trigger_type == GithubPipelineTriggerType.workflow_dispatch:
         manual_trigger_type = validate_and_extract_manual_exec_params(args.manual_trigger_type, args.tag_name)
-        publish_docker_image_manually(manual_trigger_type_param=manual_trigger_type, current_branch=current_branch,
+        publish_docker_image_manually(manual_trigger_type_param=manual_trigger_type,
+                                      will_image_be_published=publish_status,
                                       docker_image_type=image_type, tag_name=args.tag_name)
     elif pipeline_trigger_type == GithubPipelineTriggerType.push:
-        publish_docker_image_on_push(image_type, args.github_ref, current_branch)
+        publish_docker_image_on_push(image_type, args.github_ref, publish_status)
     else:
-        publish_docker_image_on_schedule(image_type, current_branch)
+        publish_docker_image_on_schedule(image_type, publish_status)

packaging_automation/tests/test_common_tool_methods.py

@@ -8,15 +8,34 @@
 
 from .test_utils import generate_new_gpg_key
 from ..common_tool_methods import (
-    find_nth_occurrence_position, is_major_release,
-    str_array_to_str, run, remove_text_with_parenthesis, get_version_details,
-    replace_line_in_file, get_upcoming_minor_version,
-    get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version,
-    get_patch_version_regex, append_line_in_file, prepend_line_in_file, remote_branch_exists, get_current_branch,
-    local_branch_exists, get_last_commit_message, get_prs_for_patch_release, filter_prs_by_label, process_template_file,
-    remove_prefix, delete_all_gpg_keys_by_name, define_rpm_public_key_to_machine,
-    delete_rpm_key_by_name, get_gpg_fingerprints_by_name, run_with_output, rpm_key_matches_summary,
-    DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER)
+    DEFAULT_ENCODING_FOR_FILE_HANDLING,
+    DEFAULT_UNICODE_ERROR_HANDLER,
+    append_line_in_file,
+    define_rpm_public_key_to_machine,
+    delete_all_gpg_keys_by_name,
+    delete_rpm_key_by_name,
+    filter_prs_by_label,
+    find_nth_matching_line_and_line_number,
+    find_nth_occurrence_position,
+    get_current_branch,
+    get_gpg_fingerprints_by_name,
+    get_last_commit_message,
+    get_minor_version,
+    get_patch_version_regex,
+    get_project_version_from_tag_name,
+    get_prs_for_patch_release,
+    get_upcoming_minor_version, get_version_details,
+    is_major_release,
+    is_tag_on_branch,
+    local_branch_exists,
+    prepend_line_in_file,
+    process_template_file,
+    remote_branch_exists,
+    remove_prefix,
+    remove_text_with_parenthesis,
+    replace_line_in_file,
+    rpm_key_matches_summary,
+    run, run_with_output, str_array_to_str)
 
 GITHUB_TOKEN = os.getenv("GH_TOKEN")
 BASE_PATH = pathlib2.Path(__file__).parents[1]
@@ -60,6 +79,14 @@ def test_get_version_details():
     assert get_version_details("10.0.1") == {"major": "10", "minor": "0", "patch": "1"}
 
 
+def test_is_tag_on_branch():
+    current_branch = get_current_branch(os.getcwd())
+    run("git checkout develop")
+    assert is_tag_on_branch("v0.8.3", "develop")
+    assert not is_tag_on_branch("v1.8.3", "develop")
+    run(f"git checkout {current_branch}")
+
+
 def test_replace_line_in_file():
     replace_str = "Summary:     Replace Test"
     copy_file_path = f"{TEST_BASE_PATH}/files/citus_copy.spec"

packaging_automation/tests/test_publish_docker.py

@@ -55,7 +55,7 @@ def test_publish_main_docker_images():
 
     try:
         run_with_output("git checkout -b docker-unit-test")
-        publish_main_docker_images(DockerImageType.latest, EXEC_PATH)
+        publish_main_docker_images(DockerImageType.latest, False)
         docker_client.images.get("citusdata/citus:latest")
     finally:
         run_with_output("git checkout master")
@@ -67,7 +67,7 @@ def test_publish_tagged_docker_images_latest():
     os.chdir("docker")
     try:
         run_with_output("git checkout -b docker-unit-test")
-        publish_tagged_docker_images(DockerImageType.latest, TAG_NAME, EXEC_PATH)
+        publish_tagged_docker_images(DockerImageType.latest, "v10.0.3", False)
         docker_client.images.get("citusdata/citus:10")
         docker_client.images.get("citusdata/citus:10.0")
         docker_client.images.get("citusdata/citus:10.0.3")
@@ -81,7 +81,7 @@ def test_publish_tagged_docker_images_alpine():
     os.chdir("docker")
     try:
         run_with_output("git checkout -b docker-unit-test")
-        publish_tagged_docker_images(DockerImageType.alpine, TAG_NAME, EXEC_PATH)
+        publish_tagged_docker_images(DockerImageType.alpine, TAG_NAME, False)
         docker_client.images.get("citusdata/citus:10-alpine")
         docker_client.images.get("citusdata/citus:10.0-alpine")
         docker_client.images.get("citusdata/citus:10.0.3-alpine")
@@ -95,15 +95,13 @@ def test_publish_nightly_docker_image():
     os.chdir("docker")
     try:
         run_with_output("git checkout -b docker-unit-test")
-        publish_nightly_docker_image(NON_DEFAULT_BRANCH_NAME)
+        publish_nightly_docker_image(False)
         docker_client.images.get("citusdata/citus:nightly")
     finally:
         run_with_output("git checkout master")
         run_with_output("git branch -D docker-unit-test")
 
 
-
-
 def clear_env():
     if os.path.exists("../docker"):
         os.chdir("..")