From 6d3a443332f23896a77254398c5d7a8962781343 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 20 Sep 2021 18:58:13 +0300 Subject: [PATCH 01/13] Adds tag name check when tag is pushed --- packaging_automation/common_tool_methods.py | 24 +++++++++++++++++-- packaging_automation/publish_docker.py | 7 +++--- .../tests/test_common_tool_methods.py | 7 +++++- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b7f6e937..a719b188 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -11,7 +11,8 @@ import gnupg import pathlib2 import requests -from git import Repo +from git import Repo,GitCommandError +import git from github import Repository, PullRequest, Commit, Github from jinja2 import Environment, FileSystemLoader from parameters_validation import validate_parameters @@ -299,10 +300,29 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: return has_match +def is_tag_on_branch(tag_name: str, branch_name: str): + g = git.Git(os.getcwd()) + try: + branches_str = g.execute(["git", "branch", "--contains", f"tags/{tag_name}"]) + branches = remove_prefix(branches_str, "*").split("\n") + if len(branches) > 0: + for branch in branches: + if branch.strip() == branch_name: + return True + return False + except GitCommandError: + return False + + + def get_current_branch(working_dir: str) -> str: repo = get_new_repo(working_dir) - return repo.active_branch.name + try: + branch_name = repo.active_branch.name + except TypeError: + branch_name = 'Detached' + return branch_name def remote_branch_exists(branch_name: str, working_dir: str) -> bool: diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index 505434cf..448e0d43 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -7,7 +7,7 @@ import pathlib2 from parameters_validation import validate_parameters -from .common_tool_methods import remove_prefix, get_current_branch +from .common_tool_methods import remove_prefix, get_current_branch, is_tag_on_branch from .common_validations import is_tag BASE_PATH = pathlib2.Path(__file__).parents[1] @@ -147,8 +147,9 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, current_branc docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'], tag=docker_image_name, path=".") - if current_branch != DEFAULT_BRANCH_NAME: - print(f"Since current branch {current_branch} is not equal to {DEFAULT_BRANCH_NAME} tags will not be pushed.") + if is_tag_on_branch(tag_name=tag_name, branch_name=DEFAULT_BRANCH_NAME): + print(f"Since default branch {DEFAULT_BRANCH_NAME} does not contain {tag_parts} ,tags will not be pushed.") + return print(f"{docker_image_type.name} image built.Now starting tagging and pushing...") for tag_part in tag_parts: tag_version_part = tag_version_part + tag_part diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 311e2445..24a05216 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -16,7 +16,7 @@ local_branch_exists, get_last_commit_message, get_prs_for_patch_release, filter_prs_by_label, process_template_file, remove_prefix, delete_all_gpg_keys_by_name, define_rpm_public_key_to_machine, delete_rpm_key_by_name, get_gpg_fingerprints_by_name, run_with_output, rpm_key_matches_summary, - DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER) + DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER, is_tag_on_branch) GITHUB_TOKEN = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[1] @@ -60,6 +60,11 @@ def test_get_version_details(): assert get_version_details("10.0.1") == {"major": "10", "minor": "0", "patch": "1"} +def test_is_tag_on_branch(): + assert is_tag_on_branch("v0.8.3", "develop") + assert not is_tag_on_branch("v1.8.3", "develop") + + def test_replace_line_in_file(): replace_str = "Summary: Replace Test" copy_file_path = f"{TEST_BASE_PATH}/files/citus_copy.spec" From e45bd7e4fa50bdfc40c2ecbc72f1253652477fe3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 10:55:31 +0300 Subject: [PATCH 02/13] Adds tag name check when tag is pushed --- packaging_automation/common_tool_methods.py | 12 ++-- packaging_automation/publish_docker.py | 68 ++++++++++++------- .../tests/test_publish_docker.py | 10 ++- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index a719b188..47690bbe 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -11,7 +11,7 @@ import gnupg import pathlib2 import requests -from git import Repo,GitCommandError +from git import Repo, GitCommandError import git from github import Repository, PullRequest, Commit, Github from jinja2 import Environment, FileSystemLoader @@ -300,6 +300,7 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: return has_match + def is_tag_on_branch(tag_name: str, branch_name: str): g = git.Git(os.getcwd()) try: @@ -314,15 +315,10 @@ def is_tag_on_branch(tag_name: str, branch_name: str): return False - - def get_current_branch(working_dir: str) -> str: repo = get_new_repo(working_dir) - try: - branch_name = repo.active_branch.name - except TypeError: - branch_name = 'Detached' - return branch_name + + return repo.active_branch.name def remote_branch_exists(branch_name: str, working_dir: str) -> bool: diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index 448e0d43..f1f82d26 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -93,53 +93,54 @@ def get_image_tag(tag_prefix: str, docker_image_type: DockerImageType) -> str: return f"{tag_prefix}{tag_suffix}" -def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, current_branch: str): +def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, will_be_published: bool): triggering_event_info, resource_name = decode_triggering_event_info(github_ref) for regular_image_type in regular_images_to_be_built(docker_image_type): if triggering_event_info == GithubTriggerEventSource.branch_push: - publish_main_docker_images(regular_image_type, current_branch) + publish_main_docker_images(regular_image_type, will_be_published) else: - publish_tagged_docker_images(regular_image_type, resource_name, current_branch) + publish_tagged_docker_images(regular_image_type, resource_name, will_be_published) -def publish_docker_image_on_schedule(docker_image_type: DockerImageType, current_branch: str, ): +def publish_docker_image_on_schedule(docker_image_type: DockerImageType, will_be_published: bool): if docker_image_type == DockerImageType.nightly: - publish_nightly_docker_image(current_branch) + publish_nightly_docker_image(will_be_published) else: for regular_image_type in regular_images_to_be_built(docker_image_type): - publish_main_docker_images(regular_image_type, current_branch) + publish_main_docker_images(regular_image_type, will_be_published) -def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, current_branch: str, +def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, will_be_published: bool, docker_image_type: DockerImageType, tag_name: str = "") -> None: if manual_trigger_type_param == ManualTriggerType.main and not tag_name: for it in regular_images_to_be_built(docker_image_type): - publish_main_docker_images(it, current_branch) + publish_main_docker_images(it, will_be_published) elif manual_trigger_type_param == ManualTriggerType.tags and tag_name: for it in regular_images_to_be_built(docker_image_type): - publish_tagged_docker_images(it, tag_name, current_branch) + publish_tagged_docker_images(it, tag_name) elif manual_trigger_type_param == ManualTriggerType.nightly: - publish_nightly_docker_image(current_branch) + publish_nightly_docker_image(will_be_published) -def publish_main_docker_images(docker_image_type: DockerImageType, current_branch: str): +def publish_main_docker_images(docker_image_type: DockerImageType, will_be_published: bool): print(f"Building main docker image for {docker_image_type.name}...") docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_type.name}" docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'], tag=docker_image_name, path=".") print(f"Main docker image for {docker_image_type.name} built.") - if current_branch == DEFAULT_BRANCH_NAME: + if will_be_published: print(f"Publishing main docker image for {docker_image_type.name}...") docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_type.name) print(f"Publishing main docker image for {docker_image_type.name} finished") else: + current_branch = get_current_branch(os.getcwd()) print( f"Since current branch {current_branch} is not equal to " f"{DEFAULT_BRANCH_NAME} {docker_image_name} will not be pushed.") -def publish_tagged_docker_images(docker_image_type, tag_name: str, current_branch: str): +def publish_tagged_docker_images(docker_image_type, tag_name: str, will_be_published: bool): print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name}...") tag_parts = decode_tag_parts(tag_name) tag_version_part = "" @@ -147,7 +148,7 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, current_branc docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'], tag=docker_image_name, path=".") - if is_tag_on_branch(tag_name=tag_name, branch_name=DEFAULT_BRANCH_NAME): + if will_be_published: print(f"Since default branch {DEFAULT_BRANCH_NAME} does not contain {tag_parts} ,tags will not be pushed.") return print(f"{docker_image_type.name} image built.Now starting tagging and pushing...") @@ -158,16 +159,15 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, current_branc docker_api_client.tag(docker_image_name, docker_image_name, image_tag) print(f"Tagging {docker_image_name} with the tag {image_tag} finished.") - if current_branch == DEFAULT_BRANCH_NAME: - print(f"Pushing {docker_image_name} with the tag {image_tag}...") - docker_client.images.push(DOCKER_IMAGE_NAME, tag=image_tag) - print(f"Pushing {docker_image_name} with the tag {image_tag} finished") + print(f"Pushing {docker_image_name} with the tag {image_tag}...") + docker_client.images.push(DOCKER_IMAGE_NAME, tag=image_tag) + print(f"Pushing {docker_image_name} with the tag {image_tag} finished") tag_version_part = tag_version_part + "." print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name} finished.") -def publish_nightly_docker_image(current_branch: str): +def publish_nightly_docker_image(will_be_published: bool): print("Building nightly image...") docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_info_dict[DockerImageType.nightly]['docker-tag']}" docker_client.images.build(dockerfile=docker_image_info_dict[DockerImageType.nightly]['file-name'], @@ -175,7 +175,7 @@ def publish_nightly_docker_image(current_branch: str): path=".") print("Nightly image build finished.") - if current_branch == DEFAULT_BRANCH_NAME: + if will_be_published: print("Pushing nightly image...") docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_info_dict[DockerImageType.nightly]['docker-tag']) print("Nightly image push finished.") @@ -217,23 +217,41 @@ def validate_and_extract_manual_exec_params(manual_trigger_type_param: str, tag_ return manual_trigger_type_param +def get_image_publish_status(github_ref: str, is_test: bool): + if is_test: + return False + triggering_event_info, resource_name = decode_triggering_event_info(github_ref) + if triggering_event_info == GithubTriggerEventSource.tag_push: + if not is_tag_on_branch(tag_name=resource_name, branch_name=DEFAULT_BRANCH_NAME): + return False + else: + return True + current_branch = get_current_branch(os.getcwd()) + if current_branch != DEFAULT_BRANCH_NAME: + return False + return True + + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--github_ref') + parser.add_argument('--github_ref', required=True) parser.add_argument('--pipeline_trigger_type', choices=[e.name for e in GithubPipelineTriggerType], required=True) parser.add_argument('--tag_name', nargs='?', default="") parser.add_argument('--manual_trigger_type', choices=[e.name for e in ManualTriggerType]) parser.add_argument('--image_type', choices=[e.name for e in DockerImageType]) + parser.add_argument('--is_test', action="store_true") args = parser.parse_args() pipeline_trigger_type, image_type = validate_and_extract_general_parameters(args.image_type, args.pipeline_trigger_type) - current_branch = get_current_branch(os.getcwd()) + + publish_status = get_image_publish_status(args.github_ref, args.is_test) if pipeline_trigger_type == GithubPipelineTriggerType.workflow_dispatch: manual_trigger_type = validate_and_extract_manual_exec_params(args.manual_trigger_type, args.tag_name) - publish_docker_image_manually(manual_trigger_type_param=manual_trigger_type, current_branch=current_branch, + publish_docker_image_manually(manual_trigger_type_param=manual_trigger_type, + will_be_published=publish_status, docker_image_type=image_type, tag_name=args.tag_name) elif pipeline_trigger_type == GithubPipelineTriggerType.push: - publish_docker_image_on_push(image_type, args.github_ref, current_branch) + publish_docker_image_on_push(image_type, args.github_ref, publish_status) else: - publish_docker_image_on_schedule(image_type, current_branch) + publish_docker_image_on_schedule(image_type, publish_status) diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 846c5edf..820b8bcf 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -55,7 +55,7 @@ def test_publish_main_docker_images(): try: run_with_output("git checkout -b docker-unit-test") - publish_main_docker_images(DockerImageType.latest, EXEC_PATH) + publish_main_docker_images(DockerImageType.latest, False) docker_client.images.get("citusdata/citus:latest") finally: run_with_output("git checkout master") @@ -67,7 +67,7 @@ def test_publish_tagged_docker_images_latest(): os.chdir("docker") try: run_with_output("git checkout -b docker-unit-test") - publish_tagged_docker_images(DockerImageType.latest, TAG_NAME, EXEC_PATH) + publish_tagged_docker_images(DockerImageType.latest, "v10.0.3", False) docker_client.images.get("citusdata/citus:10") docker_client.images.get("citusdata/citus:10.0") docker_client.images.get("citusdata/citus:10.0.3") @@ -81,7 +81,7 @@ def test_publish_tagged_docker_images_alpine(): os.chdir("docker") try: run_with_output("git checkout -b docker-unit-test") - publish_tagged_docker_images(DockerImageType.alpine, TAG_NAME, EXEC_PATH) + publish_tagged_docker_images(DockerImageType.alpine, TAG_NAME, False) docker_client.images.get("citusdata/citus:10-alpine") docker_client.images.get("citusdata/citus:10.0-alpine") docker_client.images.get("citusdata/citus:10.0.3-alpine") @@ -95,15 +95,13 @@ def test_publish_nightly_docker_image(): os.chdir("docker") try: run_with_output("git checkout -b docker-unit-test") - publish_nightly_docker_image(NON_DEFAULT_BRANCH_NAME) + publish_nightly_docker_image(False) docker_client.images.get("citusdata/citus:nightly") finally: run_with_output("git checkout master") run_with_output("git branch -D docker-unit-test") - - def clear_env(): if os.path.exists("../docker"): os.chdir("..") From d00d838416e7f902116cf75a3620ae387da81925 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 10:57:50 +0300 Subject: [PATCH 03/13] Fixes missing argument for manual tagging --- packaging_automation/publish_docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index f1f82d26..0041b165 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -117,7 +117,7 @@ def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, publish_main_docker_images(it, will_be_published) elif manual_trigger_type_param == ManualTriggerType.tags and tag_name: for it in regular_images_to_be_built(docker_image_type): - publish_tagged_docker_images(it, tag_name) + publish_tagged_docker_images(it, tag_name, will_be_published) elif manual_trigger_type_param == ManualTriggerType.nightly: publish_nightly_docker_image(will_be_published) From 056559427124088706ae4ef7f4e2804da253fde6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 11:19:39 +0300 Subject: [PATCH 04/13] Details some messages --- packaging_automation/publish_docker.py | 49 ++++++++++++++------------ 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index 0041b165..23a0884d 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -54,7 +54,7 @@ class ScheduleType(Enum): "schedule-type": ScheduleType.regular}, DockerImageType.nightly: {"file-name": "nightly/Dockerfile", "docker-tag": "nightly", "schedule-type": ScheduleType.nightly}} -DOCKER_IMAGE_NAME = "citusdata/citus" +DOCKER_IMAGE_NAME = "gurkanindibay/citus" docker_client = docker.from_env() @@ -93,54 +93,55 @@ def get_image_tag(tag_prefix: str, docker_image_type: DockerImageType) -> str: return f"{tag_prefix}{tag_suffix}" -def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, will_be_published: bool): +def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, publish: bool): triggering_event_info, resource_name = decode_triggering_event_info(github_ref) for regular_image_type in regular_images_to_be_built(docker_image_type): if triggering_event_info == GithubTriggerEventSource.branch_push: - publish_main_docker_images(regular_image_type, will_be_published) + publish_main_docker_images(regular_image_type, publish) else: - publish_tagged_docker_images(regular_image_type, resource_name, will_be_published) + publish_tagged_docker_images(regular_image_type, resource_name, publish) -def publish_docker_image_on_schedule(docker_image_type: DockerImageType, will_be_published: bool): +def publish_docker_image_on_schedule(docker_image_type: DockerImageType, publish: bool): if docker_image_type == DockerImageType.nightly: - publish_nightly_docker_image(will_be_published) + publish_nightly_docker_image(publish) else: for regular_image_type in regular_images_to_be_built(docker_image_type): - publish_main_docker_images(regular_image_type, will_be_published) + publish_main_docker_images(regular_image_type, publish) -def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, will_be_published: bool, +def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, publish: bool, docker_image_type: DockerImageType, tag_name: str = "") -> None: if manual_trigger_type_param == ManualTriggerType.main and not tag_name: for it in regular_images_to_be_built(docker_image_type): - publish_main_docker_images(it, will_be_published) + publish_main_docker_images(it, publish) elif manual_trigger_type_param == ManualTriggerType.tags and tag_name: for it in regular_images_to_be_built(docker_image_type): - publish_tagged_docker_images(it, tag_name, will_be_published) + publish_tagged_docker_images(it, tag_name, publish) elif manual_trigger_type_param == ManualTriggerType.nightly: - publish_nightly_docker_image(will_be_published) + publish_nightly_docker_image(publish) -def publish_main_docker_images(docker_image_type: DockerImageType, will_be_published: bool): +def publish_main_docker_images(docker_image_type: DockerImageType, publish: bool): print(f"Building main docker image for {docker_image_type.name}...") docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_type.name}" docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'], tag=docker_image_name, path=".") print(f"Main docker image for {docker_image_type.name} built.") - if will_be_published: + if publish: print(f"Publishing main docker image for {docker_image_type.name}...") docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_type.name) print(f"Publishing main docker image for {docker_image_type.name} finished") else: current_branch = get_current_branch(os.getcwd()) - print( - f"Since current branch {current_branch} is not equal to " - f"{DEFAULT_BRANCH_NAME} {docker_image_name} will not be pushed.") + if current_branch != DEFAULT_BRANCH_NAME: + print( + f"Since current branch {current_branch} is not equal to " + f"{DEFAULT_BRANCH_NAME} {docker_image_name} will not be pushed.") -def publish_tagged_docker_images(docker_image_type, tag_name: str, will_be_published: bool): +def publish_tagged_docker_images(docker_image_type, tag_name: str, publish: bool): print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name}...") tag_parts = decode_tag_parts(tag_name) tag_version_part = "" @@ -148,8 +149,8 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, will_be_publi docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'], tag=docker_image_name, path=".") - if will_be_published: - print(f"Since default branch {DEFAULT_BRANCH_NAME} does not contain {tag_parts} ,tags will not be pushed.") + if not publish: + print(f"Tags will not be pushed since publish flag is false. ") return print(f"{docker_image_type.name} image built.Now starting tagging and pushing...") for tag_part in tag_parts: @@ -167,7 +168,7 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, will_be_publi print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name} finished.") -def publish_nightly_docker_image(will_be_published: bool): +def publish_nightly_docker_image(publish: bool): print("Building nightly image...") docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_info_dict[DockerImageType.nightly]['docker-tag']}" docker_client.images.build(dockerfile=docker_image_info_dict[DockerImageType.nightly]['file-name'], @@ -175,10 +176,12 @@ def publish_nightly_docker_image(will_be_published: bool): path=".") print("Nightly image build finished.") - if will_be_published: + if publish: print("Pushing nightly image...") docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_info_dict[DockerImageType.nightly]['docker-tag']) print("Nightly image push finished.") + else: + print("Nightly image will not be pushed since publish flag is false") def validate_and_extract_general_parameters(docker_image_type_param: str, pipeline_trigger_type_param: str) -> Tuple[ @@ -244,12 +247,14 @@ def get_image_publish_status(github_ref: str, is_test: bool): pipeline_trigger_type, image_type = validate_and_extract_general_parameters(args.image_type, args.pipeline_trigger_type) + if args.is_test: + print("Script is working in test mode. Images will not be published") publish_status = get_image_publish_status(args.github_ref, args.is_test) if pipeline_trigger_type == GithubPipelineTriggerType.workflow_dispatch: manual_trigger_type = validate_and_extract_manual_exec_params(args.manual_trigger_type, args.tag_name) publish_docker_image_manually(manual_trigger_type_param=manual_trigger_type, - will_be_published=publish_status, + publish=publish_status, docker_image_type=image_type, tag_name=args.tag_name) elif pipeline_trigger_type == GithubPipelineTriggerType.push: publish_docker_image_on_push(image_type, args.github_ref, publish_status) From 5032a007d62f0cca0fd3341030f26e9b7a73971b Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 11:30:56 +0300 Subject: [PATCH 05/13] Moves publish flag check into tag block --- packaging_automation/publish_docker.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index 23a0884d..5fffe3e4 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -149,9 +149,6 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, publish: bool docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'], tag=docker_image_name, path=".") - if not publish: - print(f"Tags will not be pushed since publish flag is false. ") - return print(f"{docker_image_type.name} image built.Now starting tagging and pushing...") for tag_part in tag_parts: tag_version_part = tag_version_part + tag_part @@ -159,10 +156,12 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, publish: bool print(f"Tagging {docker_image_name} with the tag {image_tag}...") docker_api_client.tag(docker_image_name, docker_image_name, image_tag) print(f"Tagging {docker_image_name} with the tag {image_tag} finished.") - - print(f"Pushing {docker_image_name} with the tag {image_tag}...") - docker_client.images.push(DOCKER_IMAGE_NAME, tag=image_tag) - print(f"Pushing {docker_image_name} with the tag {image_tag} finished") + if publish: + print(f"Pushing {docker_image_name} with the tag {image_tag}...") + docker_client.images.push(DOCKER_IMAGE_NAME, tag=image_tag) + print(f"Pushing {docker_image_name} with the tag {image_tag} finished") + else: + print(f"Skipped pushing {docker_image_type} with the tag {image_tag} since publish flag is false") tag_version_part = tag_version_part + "." print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name} finished.") From bbc9b1c1deb3a5a12467e0986e007d748877d3fd Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 11:34:37 +0300 Subject: [PATCH 06/13] Fixes DOCKER_IMAGE_NAME --- packaging_automation/publish_docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index 5fffe3e4..5fc37ae5 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -54,7 +54,7 @@ class ScheduleType(Enum): "schedule-type": ScheduleType.regular}, DockerImageType.nightly: {"file-name": "nightly/Dockerfile", "docker-tag": "nightly", "schedule-type": ScheduleType.nightly}} -DOCKER_IMAGE_NAME = "gurkanindibay/citus" +DOCKER_IMAGE_NAME = "citusdata/citus" docker_client = docker.from_env() From c938e46758e4b385d980f865aab462172edc9086 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 11:35:53 +0300 Subject: [PATCH 07/13] Fixes static code analysis checks --- packaging_automation/publish_docker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index 5fc37ae5..e2e1a1b6 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -226,8 +226,7 @@ def get_image_publish_status(github_ref: str, is_test: bool): if triggering_event_info == GithubTriggerEventSource.tag_push: if not is_tag_on_branch(tag_name=resource_name, branch_name=DEFAULT_BRANCH_NAME): return False - else: - return True + return True current_branch = get_current_branch(os.getcwd()) if current_branch != DEFAULT_BRANCH_NAME: return False From a00ec59935fc71b4a4f4dbca30855d159cae7dbf Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 11:51:19 +0300 Subject: [PATCH 08/13] Changes fetch depth 0 to get all branches --- .github/workflows/tool-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 677c91d2..421a20cb 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -29,6 +29,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Define git credentials run: git config --global user.email "${MICROSOFT_EMAIL}"&& git config --global user.name "${USER_NAME}" From a3176efa05585433975759f6e91ee1ae8418d1e7 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 12:09:50 +0300 Subject: [PATCH 09/13] Adds debug messages to detect unit test errors --- .github/workflows/tool-tests.yml | 6 +++--- packaging_automation/common_tool_methods.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 421a20cb..e800539f 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -44,6 +44,9 @@ jobs: - name: Run static code analysis run: python -m prospector + - name: Unit tests for "Common tools" + run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py + - name: Unit tests for "Update Package Properties" run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py @@ -56,8 +59,5 @@ jobs: - name: Unit tests for "Update Pgxn" run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py - - name: Unit tests for "Common tools" - run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py - - name: Packaging Warning Handler run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 47690bbe..9ad4b9a4 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -306,12 +306,14 @@ def is_tag_on_branch(tag_name: str, branch_name: str): try: branches_str = g.execute(["git", "branch", "--contains", f"tags/{tag_name}"]) branches = remove_prefix(branches_str, "*").split("\n") + print("Branches str:" + branches_str) if len(branches) > 0: for branch in branches: if branch.strip() == branch_name: return True return False - except GitCommandError: + except GitCommandError as e: + print("Error:" + str(e)) return False From a1c290b3eb686f1a13f9d53492cc4d2467038d2e Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 21 Sep 2021 12:17:44 +0300 Subject: [PATCH 10/13] Adds debug messages to detect unit test errors --- packaging_automation/tests/test_common_tool_methods.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 24a05216..c67f8f0b 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -61,8 +61,11 @@ def test_get_version_details(): def test_is_tag_on_branch(): + current_branch = get_current_branch(os.getcwd()) + run("git checkout develop") assert is_tag_on_branch("v0.8.3", "develop") assert not is_tag_on_branch("v1.8.3", "develop") + run(f"git checkout {current_branch}") def test_replace_line_in_file(): From 1215a2c7a429e65ee7d92232f5d13a41f44604f5 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 27 Sep 2021 17:23:20 +0300 Subject: [PATCH 11/13] Fixes review notes --- packaging_automation/common_tool_methods.py | 10 ++--- .../tests/test_common_tool_methods.py | 37 ++++++++++++++----- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 9ad4b9a4..a69b0d16 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -1,19 +1,17 @@ -import base64 import os import re import shlex import subprocess from datetime import datetime from enum import Enum -from typing import Dict, List -from typing import Tuple +from typing import Dict, List,Tuple +import git import gnupg import pathlib2 import requests -from git import Repo, GitCommandError -import git -from github import Repository, PullRequest, Commit, Github +from git import GitCommandError,Repo +from github import Commit, Github, PullRequest,Repository from jinja2 import Environment, FileSystemLoader from parameters_validation import validate_parameters diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index c67f8f0b..4d7271c2 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -8,15 +8,34 @@ from .test_utils import generate_new_gpg_key from ..common_tool_methods import ( - find_nth_occurrence_position, is_major_release, - str_array_to_str, run, remove_text_with_parenthesis, get_version_details, - replace_line_in_file, get_upcoming_minor_version, - get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, - get_patch_version_regex, append_line_in_file, prepend_line_in_file, remote_branch_exists, get_current_branch, - local_branch_exists, get_last_commit_message, get_prs_for_patch_release, filter_prs_by_label, process_template_file, - remove_prefix, delete_all_gpg_keys_by_name, define_rpm_public_key_to_machine, - delete_rpm_key_by_name, get_gpg_fingerprints_by_name, run_with_output, rpm_key_matches_summary, - DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER, is_tag_on_branch) + DEFAULT_ENCODING_FOR_FILE_HANDLING, + DEFAULT_UNICODE_ERROR_HANDLER, + append_line_in_file, + define_rpm_public_key_to_machine, + delete_all_gpg_keys_by_name, + delete_rpm_key_by_name, + filter_prs_by_label, + find_nth_matching_line_and_line_number, + find_nth_occurrence_position, + get_current_branch, + get_gpg_fingerprints_by_name, + get_last_commit_message, + get_minor_version, + get_patch_version_regex, + get_project_version_from_tag_name, + get_prs_for_patch_release, + get_upcoming_minor_version, get_version_details, + is_major_release, + is_tag_on_branch, + local_branch_exists, + prepend_line_in_file, + process_template_file, + remote_branch_exists, + remove_prefix, + remove_text_with_parenthesis, + replace_line_in_file, + rpm_key_matches_summary, + run, run_with_output, str_array_to_str) GITHUB_TOKEN = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[1] From c0675af48411f53933fa96d45d589875bf46c3da Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 27 Sep 2021 17:38:51 +0300 Subject: [PATCH 12/13] Changes publish parameter --- packaging_automation/publish_docker.py | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index e2e1a1b6..28e553e4 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -93,43 +93,43 @@ def get_image_tag(tag_prefix: str, docker_image_type: DockerImageType) -> str: return f"{tag_prefix}{tag_suffix}" -def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, publish: bool): +def publish_docker_image_on_push(docker_image_type: DockerImageType, github_ref: str, will_image_be_published: bool): triggering_event_info, resource_name = decode_triggering_event_info(github_ref) for regular_image_type in regular_images_to_be_built(docker_image_type): if triggering_event_info == GithubTriggerEventSource.branch_push: - publish_main_docker_images(regular_image_type, publish) + publish_main_docker_images(regular_image_type, will_image_be_published) else: - publish_tagged_docker_images(regular_image_type, resource_name, publish) + publish_tagged_docker_images(regular_image_type, resource_name, will_image_be_published) -def publish_docker_image_on_schedule(docker_image_type: DockerImageType, publish: bool): +def publish_docker_image_on_schedule(docker_image_type: DockerImageType, will_image_be_published: bool): if docker_image_type == DockerImageType.nightly: - publish_nightly_docker_image(publish) + publish_nightly_docker_image(will_image_be_published) else: for regular_image_type in regular_images_to_be_built(docker_image_type): - publish_main_docker_images(regular_image_type, publish) + publish_main_docker_images(regular_image_type, will_image_be_published) -def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, publish: bool, +def publish_docker_image_manually(manual_trigger_type_param: ManualTriggerType, will_image_be_published: bool, docker_image_type: DockerImageType, tag_name: str = "") -> None: if manual_trigger_type_param == ManualTriggerType.main and not tag_name: for it in regular_images_to_be_built(docker_image_type): - publish_main_docker_images(it, publish) + publish_main_docker_images(it, will_image_be_published) elif manual_trigger_type_param == ManualTriggerType.tags and tag_name: for it in regular_images_to_be_built(docker_image_type): - publish_tagged_docker_images(it, tag_name, publish) + publish_tagged_docker_images(it, tag_name, will_image_be_published) elif manual_trigger_type_param == ManualTriggerType.nightly: - publish_nightly_docker_image(publish) + publish_nightly_docker_image(will_image_be_published) -def publish_main_docker_images(docker_image_type: DockerImageType, publish: bool): +def publish_main_docker_images(docker_image_type: DockerImageType, will_image_be_published: bool): print(f"Building main docker image for {docker_image_type.name}...") docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_type.name}" docker_client.images.build(dockerfile=docker_image_info_dict[docker_image_type]['file-name'], tag=docker_image_name, path=".") print(f"Main docker image for {docker_image_type.name} built.") - if publish: + if will_image_be_published: print(f"Publishing main docker image for {docker_image_type.name}...") docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_type.name) print(f"Publishing main docker image for {docker_image_type.name} finished") @@ -141,7 +141,7 @@ def publish_main_docker_images(docker_image_type: DockerImageType, publish: bool f"{DEFAULT_BRANCH_NAME} {docker_image_name} will not be pushed.") -def publish_tagged_docker_images(docker_image_type, tag_name: str, publish: bool): +def publish_tagged_docker_images(docker_image_type, tag_name: str, will_image_be_published: bool): print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name}...") tag_parts = decode_tag_parts(tag_name) tag_version_part = "" @@ -156,18 +156,18 @@ def publish_tagged_docker_images(docker_image_type, tag_name: str, publish: bool print(f"Tagging {docker_image_name} with the tag {image_tag}...") docker_api_client.tag(docker_image_name, docker_image_name, image_tag) print(f"Tagging {docker_image_name} with the tag {image_tag} finished.") - if publish: + if will_image_be_published: print(f"Pushing {docker_image_name} with the tag {image_tag}...") docker_client.images.push(DOCKER_IMAGE_NAME, tag=image_tag) print(f"Pushing {docker_image_name} with the tag {image_tag} finished") else: - print(f"Skipped pushing {docker_image_type} with the tag {image_tag} since publish flag is false") + print(f"Skipped pushing {docker_image_type} with the tag {image_tag} since will_image_be_published flag is false") tag_version_part = tag_version_part + "." print(f"Building and publishing tagged image {docker_image_type.name} for tag {tag_name} finished.") -def publish_nightly_docker_image(publish: bool): +def publish_nightly_docker_image(will_image_be_published: bool): print("Building nightly image...") docker_image_name = f"{DOCKER_IMAGE_NAME}:{docker_image_info_dict[DockerImageType.nightly]['docker-tag']}" docker_client.images.build(dockerfile=docker_image_info_dict[DockerImageType.nightly]['file-name'], @@ -175,12 +175,12 @@ def publish_nightly_docker_image(publish: bool): path=".") print("Nightly image build finished.") - if publish: + if will_image_be_published: print("Pushing nightly image...") docker_client.images.push(DOCKER_IMAGE_NAME, tag=docker_image_info_dict[DockerImageType.nightly]['docker-tag']) print("Nightly image push finished.") else: - print("Nightly image will not be pushed since publish flag is false") + print("Nightly image will not be pushed since will_image_be_published flag is false") def validate_and_extract_general_parameters(docker_image_type_param: str, pipeline_trigger_type_param: str) -> Tuple[ @@ -252,7 +252,7 @@ def get_image_publish_status(github_ref: str, is_test: bool): if pipeline_trigger_type == GithubPipelineTriggerType.workflow_dispatch: manual_trigger_type = validate_and_extract_manual_exec_params(args.manual_trigger_type, args.tag_name) publish_docker_image_manually(manual_trigger_type_param=manual_trigger_type, - publish=publish_status, + will_image_be_published=publish_status, docker_image_type=image_type, tag_name=args.tag_name) elif pipeline_trigger_type == GithubPipelineTriggerType.push: publish_docker_image_on_push(image_type, args.github_ref, publish_status) From 08d184e6fd932882adea6bb1133e770fdcef7490 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 27 Sep 2021 17:42:46 +0300 Subject: [PATCH 13/13] Fixes missing import --- packaging_automation/common_tool_methods.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index a69b0d16..ba9fd2f0 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -4,14 +4,15 @@ import subprocess from datetime import datetime from enum import Enum -from typing import Dict, List,Tuple +from typing import Dict, List, Tuple +import base64 import git import gnupg import pathlib2 import requests -from git import GitCommandError,Repo -from github import Commit, Github, PullRequest,Repository +from git import GitCommandError, Repo +from github import Commit, Github, PullRequest, Repository from jinja2 import Environment, FileSystemLoader from parameters_validation import validate_parameters