diff --git a/.gitignore b/.gitignore index 65175e055dea..e8be2c5382b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -build/pmm-server-docker/RPMS/ +build/ *.swp *~ tmp/ diff --git a/.gitmodules b/.gitmodules index 8ea04b6c0957..e948f8d4225d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,14 +1,14 @@ +# PMM Client and Server Components [submodule "pmm"] path = sources/pmm/src/github.com/percona/pmm url = https://github.com/percona/pmm branch = v3 # PMM Client - [submodule "node_exporter"] path = sources/node_exporter/src/github.com/prometheus/node_exporter url = https://github.com/percona/node_exporter.git - branch = main + branch = v3 [submodule "mysqld_exporter"] path = sources/mysqld_exporter/src/github.com/percona/mysqld_exporter url = https://github.com/percona/mysqld_exporter.git @@ -39,7 +39,6 @@ branch = release-v3.5.2 # PMM Server - [submodule "grafana-dashboards"] path = sources/grafana-dashboards url = https://github.com/percona/grafana-dashboards @@ -52,6 +51,10 @@ path = sources/pmm-dump url = https://github.com/percona/pmm-dump branch = v3 +[submodule "watchtower"] + path = sources/watchtower/src/github.com/percona/watchtower + url = https://github.com/percona/watchtower + branch = main ## PMM Tests [submodule "pmm-qa"] diff --git a/Makefile b/Makefile index 9e3dae692622..e5b6723c4e70 100644 --- a/Makefile +++ b/Makefile @@ -14,18 +14,18 @@ help: ## Display this help message. @grep '^[a-zA-Z]' $(MAKEFILE_LIST) | \ awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}' -submodules: ## Update all sumodules . +submodules: ## Update all sumodules. git submodule update --init --remote --jobs 10 git submodule status -deps: ## Get deps from repos +deps: ## Get deps from repos. python3 ci.py -trigger: - git commit -m 'Trigger FB' --allow-empty +trigger: ## Make an empty commit to trigger the build. + git commit -m 'chore: trigger FB' --allow-empty git push -prepare: ## Create new FB (new style) +prepare: ## Create a new FB (new style). python3 ci.py -g --prepare $(RUN_ARGS) clean: ## Clean build results. @@ -36,8 +36,8 @@ purge: ## Clean cache and leftovers. Please run this when s git submodule update git submodule foreach 'git reset --hard && git clean -xdff' -fb: ## Creates feature build branch. - # Usage: make fb mainBranch=v3 featureBranch=PMM-XXXX-name submodules="pmm pmm-managed" +fb: ## Create a feature build branch. + # Usage: make fb mainBranch=v3 featureBranch=PMM-XXXX-branch-name submodules="pmm pmm-managed" $(eval MAIN_BRANCH = $(or $(mainBranch),v3)) git checkout $(MAIN_BRANCH) make purge diff --git a/README.md b/README.md index 5dc10f5269ae..64511dda512c 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ to our build system managed by Jenkins as it helps pull the right branches from # Installation of dependencies -If you build with Python's script then you need to install the dependencies: +If you intend to build PMM with 'ci.py' Python script, then you need to install the dependencies: -``` +```shell pip install -r requirements.txt ``` @@ -18,29 +18,29 @@ To create a feature build (FB) you have to edit `ci.yml` and specify the branche ```yaml deps: - name: pmm - branch: PMM-0000-fix-everything + branch: PMM-0000-feature-branch - name: pmm-qa - branch: PMM-0000-fix-everything-and-even-more + branch: PMM-0000-bugfix-branch ``` -To build from a fork, you need to specify `url` for the dependency, for example: +To build from a fork, you need to additionally specify `url` for the dependency, for example: ```yaml deps: - name: pmm-server url: https://github.com//pmm-server - branch: PMM-0000-fix-everything + branch: PMM-0000-feature-branch ``` -Next, you will commit changes to git and push them to the repo: +Next, commit your changes to git and push them to the repo: -``` +```shell git add ci.yml git commit -m 'use custom branches' git push ``` -Whenever you commit and push to a feature branch, a Jenkins job will be triggered and it will start building your feature. You can follow its progress right from the PR's actions (at the bottom of each PR). +Whenever you commit and push to a feature branch, a Jenkins job will be triggered, which will start building your feature. You can follow its progress by following a github workflow link displayed at the bottom of each PR. ## Using a Personal Access Token (PAT) @@ -54,21 +54,23 @@ The token requires the following permissions: It is recommended to set an expiration date for your token. -if you use zsh: +To make the Github token available to `ci.py`, add the following lines to your shell script: + +if you use `zsh` shell: ```console echo 'export GITHUB_API_TOKEN=********' >> ~/.zshrc source ~/.zshrc ``` -if you use bash: +if you use `bash` shell: ```console echo 'export GITHUB_API_TOKEN=********' >> ~/.bash_profile source ~/.bash_profile ``` -NOTE: Please make sure you don't commit your PAT to github. Should the PAT accidentally leak out, please revoke it asap and re-create it. +NOTE: Please make sure you DO NOT commit your PAT to github. Should the PAT accidentally leak out, please revoke it asap and re-create it. ## FAQ @@ -77,15 +79,15 @@ NOTE: Please make sure you don't commit your PAT to github. Should the PAT accid If you use the same branch name in all repos then you can run: ```console -make prepare +make prepare ``` -Branches with "you branch name" will be used for all repos or the default branch (usually called `main`) if the branch with this name isn't found in the repo. +Branch "your-branch-name" will be used for every submodule repo. If not found, the repo's default branch (usually `main`) will be used. -If you want to create a FB from a fork, you can pass an environment variable "FORK_OWNER" which should be equal to your username in github and run: +If you want to create a FB from a fork, you can pass an environment variable "FORK_OWNER", which should be set to your github username: ```console -FORK_OWNER= make prepare +FORK_OWNER= make prepare ``` ### I got an error "...branch has no upstream branch" diff --git a/ci-default.yml b/ci-default.yml deleted file mode 100644 index 9bf5643d53f0..000000000000 --- a/ci-default.yml +++ /dev/null @@ -1,89 +0,0 @@ -deps: - # COMMON - - name: pmm - branch: v3 - path: sources/pmm/src/github.com/percona/pmm - url: https://github.com/percona/pmm - component: common - - # CLIENT - - name: node_exporter - branch: main - path: sources/node_exporter/src/github.com/prometheus/node_exporter - url: https://github.com/percona/node_exporter - component: client - - - name: mysqld_exporter - branch: main - path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter - url: https://github.com/percona/mysqld_exporter - component: client - - - name: mongodb_exporter - branch: main - path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter - url: https://github.com/percona/mongodb_exporter - component: client - - - name: postgres_exporter - branch: main - path: sources/postgres_exporter/src/github.com/percona/postgres_exporter - url: https://github.com/percona/postgres_exporter - component: client - - - name: proxysql_exporter - branch: main - path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter - url: https://github.com/percona/proxysql_exporter - component: client - - - name: rds_exporter - branch: main - path: sources/rds_exporter/src/github.com/percona/rds_exporter - url: https://github.com/percona/rds_exporter - component: client - - - name: azure_metrics_exporter - branch: main - path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter - url: https://github.com/percona/azure_metrics_exporter - component: client - - - name: percona-toolkit - branch: release-v3.5.2 - path: sources/percona-toolkit/src/github.com/percona/percona-toolkit - url: https://github.com/percona/percona-toolkit - component: client - - # SERVER - - - name: grafana-dashboards - branch: v3 - path: sources/grafana-dashboards - url: https://github.com/percona/grafana-dashboards - component: server - - - name: grafana - branch: v3 - path: sources/grafana/src/github.com/grafana/grafana - url: https://github.com/percona/grafana - component: server - - - name: pmm-dump - branch: main - path: sources/pmm-dump - url: https://github.com/percona/pmm-dump - component: server - - # QA - - name: pmm-qa - branch: v3 - path: sources/pmm-qa/src/github.com/percona/pmm-qa - url: https://github.com/percona/pmm-qa - component: qa - - - name: pmm-ui-tests - branch: v3 - path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests - url: https://github.com/percona/pmm-ui-tests - component: qa diff --git a/ci.py b/ci.py index 36b4aa0540f8..13c477dec309 100644 --- a/ci.py +++ b/ci.py @@ -17,7 +17,7 @@ logging.basicConfig(stream=sys.stdout, format='[%(levelname)s] %(asctime)s: %(message)s', level=logging.INFO) -YAML_CONFIG = 'ci-default.yml' +YAML_EXPORT_CONFIG = 'gitmodules.yml' YAML_CONFIG_OVERRIDE = 'ci.yml' SUBMODULES_CONFIG = '.gitmodules' GIT_SOURCES_FILE = '.git-sources' @@ -31,6 +31,8 @@ class Builder(): rootdir = check_output(['git', 'rev-parse', '--show-toplevel']).decode('utf-8').strip() def __init__(self): + self.config_source = SUBMODULES_CONFIG + self.config_override = self.read_config_override() self.config = self.read_config() @@ -42,8 +44,17 @@ def read_config_override(self): return yaml.load(f, Loader=yaml.FullLoader) def read_config(self): - with open(YAML_CONFIG, 'r') as f: - return yaml.load(f, Loader=yaml.FullLoader) + config = configparser.ConfigParser() + config.read(self.config_source) + + submodules = [] + for s in config.sections(): + submodules_name = s.split('"')[1] + submodules_info = dict(config.items(s)) + submodules_info['name'] = submodules_name + + submodules.append(submodules_info) + return {'deps': submodules} def write_custom_config(self, config): with open(YAML_CONFIG_OVERRIDE, 'w') as f: @@ -65,6 +76,16 @@ def merge_configs(self): f'Can"t find {override_dep["name"]} repo from ci.yml in the list of repos in ci-default.yml') sys.exit(1) + # To test the merge, run `python ./ci.py --convert` + def export_gitmodules_to_yaml(self, target=YAML_EXPORT_CONFIG): + yaml_config = Path(target) + if yaml_config.is_file(): + logging.warning('File {} already exists!'.format(target)) + sys.exit(1) + with open(target, 'w') as f: + yaml.dump(self.config, f, sort_keys=False) + sys.exit(0) + def get_global_branches(self, target_branch_name): found_branches = {} github_api = Github(GITHUB_TOKEN) @@ -158,7 +179,7 @@ def create_fb_branch(self, branch_name, global_repo=False): f'Pull Request was created: https://github.com/Percona-Lab/pmm-submodules/pull/{pr.number}') else: logging.info( - f'Pull request already exist: https://github.com/Percona-Lab/pmm-submodules/pull/{pr[0].number}') + f'Pull request already exists: https://github.com/Percona-Lab/pmm-submodules/pull/{pr[0].number}') else: logging.info('Branch was created') logging.info( @@ -232,60 +253,24 @@ def check_deps(self): pull.create_issue_comment(outdated_branches_message) sys.exit(1) - def create_release(self): - pass - - def create_tags(self): - pass - def validate_config(self): for dep in self.config['deps']: if not os.path.abspath(dep['path']).startswith(os.getcwd()): logging.error(f'For dependency [{dep["name"]} -> {os.path.abspath(dep["path"])}] ' - f'path must be in working directory [{os.getcwd()}]') + f'the path must be located within the working directory [{os.getcwd()}]') sys.exit(1) -class Converter: - def __init__(self, origin=SUBMODULES_CONFIG, target=YAML_CONFIG): - self.origin = origin - self.target = target - self.submodules = self.get_list_of_submodules() - self.convert_gitmodules_to_yaml() - - def get_list_of_submodules(self): - config = configparser.ConfigParser() - config.read(self.origin) - - submodules = [] - for s in config.sections(): - submodules_name = s.split('"')[1] - submodules_info = dict(config.items(s)) - submodules_info['name'] = submodules_name - - submodules.append(submodules_info) - return {'deps': submodules} - - def convert_gitmodules_to_yaml(self): - yaml_config = Path(self.target) - if yaml_config.is_file(): - logging.warning('File {} already exist!'.format(self.target)) - sys.exit(1) - with open(self.target, 'w') as f: - yaml.dump(self.submodules, f, sort_keys=False) - sys.exit(0) - - def switch_branch(path, branch): - # symbolic-ref works only if we on branch. If we use commit we use rev-parse instead + # 'symbolic-ref' works only if we are on a branch. If we use a commit, we run 'rev-parse' instead. try: cur_branch = check_output('git symbolic-ref --short HEAD'.split(), cwd=path).decode().strip() except CalledProcessError: cur_branch = check_output('git rev-parse HEAD'.split(), cwd=path).decode().strip() if cur_branch != branch: branches = check_output('git ls-remote --heads origin'.split(), cwd=path) - branches = [line.split("/")[-1] - for line in branches.decode().strip().split("\n")] + branches = [line.split("/")[-1] for line in branches.decode().strip().split("\n")] + if branch in branches: print(f'Switch to branch: {branch} (from {cur_branch})') check_call(f'git remote set-branches origin {branch}'.split(), cwd=path) @@ -303,18 +288,16 @@ def main(): parser.add_argument('--prepare', help='prepare feature build') parser.add_argument('--global', '-g', dest='global_repo', help='find and use all branches with this name', action='store_true') - parser.add_argument('--convert', help='convert .gitmodules to .git-deps.yml', action='store_true') - parser.add_argument('--release', help='create release candidate') - parser.add_argument('--tags', help='create tag') - parser.add_argument('--get_branch', help='get branch name for repo') + parser.add_argument('--convert', help='convert .gitmodules config to yml', action='store_true') args = parser.parse_args() + builder = Builder() + if args.convert: - Converter() + builder.export_gitmodules_to_yaml() sys.exit(0) - builder = Builder() if args.prepare: builder.create_fb_branch(args.prepare, args.global_repo) sys.exit(0) diff --git a/requirements.txt b/requirements.txt index 1c2b51c053c8..92697e5ab31c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -pyaml==23.12.0 -GitPython==3.1.41 -PyGithub==2.1.1 +pyaml==24.9.0 +GitPython==3.1.43 +PyGithub==2.5.0 diff --git a/sources/grafana-dashboards b/sources/grafana-dashboards index 8ab359c707c1..d14f1283a159 160000 --- a/sources/grafana-dashboards +++ b/sources/grafana-dashboards @@ -1 +1 @@ -Subproject commit 8ab359c707c1f9c60e444ccf7d6293249cf8cb6d +Subproject commit d14f1283a159942d024e2323eb0e7464f6e7b4f2 diff --git a/sources/grafana/src/github.com/grafana/grafana b/sources/grafana/src/github.com/grafana/grafana index f934347564ab..a0ce41d67ec6 160000 --- a/sources/grafana/src/github.com/grafana/grafana +++ b/sources/grafana/src/github.com/grafana/grafana @@ -1 +1 @@ -Subproject commit f934347564abef97e4a6f062df74255002455218 +Subproject commit a0ce41d67ec64000ae4aeb9aeb541ebc5172265a diff --git a/sources/mongodb_exporter/src/github.com/percona/mongodb_exporter b/sources/mongodb_exporter/src/github.com/percona/mongodb_exporter index deaf10675b1c..7f0aed8ad2c4 160000 --- a/sources/mongodb_exporter/src/github.com/percona/mongodb_exporter +++ b/sources/mongodb_exporter/src/github.com/percona/mongodb_exporter @@ -1 +1 @@ -Subproject commit deaf10675b1c74ec5e4317dae4ad41cd8852ff82 +Subproject commit 7f0aed8ad2c4f544721ec8cdc40f5a250fbc8847 diff --git a/sources/mysqld_exporter/src/github.com/percona/mysqld_exporter b/sources/mysqld_exporter/src/github.com/percona/mysqld_exporter index 70740e148536..48820b70eba9 160000 --- a/sources/mysqld_exporter/src/github.com/percona/mysqld_exporter +++ b/sources/mysqld_exporter/src/github.com/percona/mysqld_exporter @@ -1 +1 @@ -Subproject commit 70740e148536e452af59602f5e9507358d229fce +Subproject commit 48820b70eba9048036b29f2ccc17e41fed66678b diff --git a/sources/node_exporter/src/github.com/prometheus/node_exporter b/sources/node_exporter/src/github.com/prometheus/node_exporter index 01de4bd38ff9..bc09327bebf8 160000 --- a/sources/node_exporter/src/github.com/prometheus/node_exporter +++ b/sources/node_exporter/src/github.com/prometheus/node_exporter @@ -1 +1 @@ -Subproject commit 01de4bd38ff91d6834655e8bb55f9e101ef54854 +Subproject commit bc09327bebf856af012d4853cf53f146b1daa045 diff --git a/sources/pmm-qa/src/github.com/percona/pmm-qa b/sources/pmm-qa/src/github.com/percona/pmm-qa index eba958909949..fbae52cd828c 160000 --- a/sources/pmm-qa/src/github.com/percona/pmm-qa +++ b/sources/pmm-qa/src/github.com/percona/pmm-qa @@ -1 +1 @@ -Subproject commit eba958909949e3280df0bd7c5dea4a76b326b701 +Subproject commit fbae52cd828c0fbdec1ef90aa125a66215681fa2 diff --git a/sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests b/sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests index f78fd1799a8b..872db101af98 160000 --- a/sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests +++ b/sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests @@ -1 +1 @@ -Subproject commit f78fd1799a8b7f3a3c8229c0dae84e13641ec0ce +Subproject commit 872db101af9894ca9f92ae787a67c66f982c3c72 diff --git a/sources/pmm/src/github.com/percona/pmm b/sources/pmm/src/github.com/percona/pmm index 43cc6db7d677..a55a6633b2bf 160000 --- a/sources/pmm/src/github.com/percona/pmm +++ b/sources/pmm/src/github.com/percona/pmm @@ -1 +1 @@ -Subproject commit 43cc6db7d677de5f089d5717f76d4b63859b3b21 +Subproject commit a55a6633b2bf7353910d4432582e13a32485ffeb diff --git a/sources/postgres_exporter/src/github.com/percona/postgres_exporter b/sources/postgres_exporter/src/github.com/percona/postgres_exporter index b4918954bfe9..bea6278da739 160000 --- a/sources/postgres_exporter/src/github.com/percona/postgres_exporter +++ b/sources/postgres_exporter/src/github.com/percona/postgres_exporter @@ -1 +1 @@ -Subproject commit b4918954bfe975c484b8058be112be20801cca2f +Subproject commit bea6278da7396f08a8ac1729e507fa2d2e1a5840 diff --git a/sources/proxysql_exporter/src/github.com/percona/proxysql_exporter b/sources/proxysql_exporter/src/github.com/percona/proxysql_exporter index de95b32bf1a9..42c7699037c1 160000 --- a/sources/proxysql_exporter/src/github.com/percona/proxysql_exporter +++ b/sources/proxysql_exporter/src/github.com/percona/proxysql_exporter @@ -1 +1 @@ -Subproject commit de95b32bf1a94d8c5a475765d039589058a7dc19 +Subproject commit 42c7699037c180f9cecf01de004bf902093a9e23 diff --git a/sources/rds_exporter/src/github.com/percona/rds_exporter b/sources/rds_exporter/src/github.com/percona/rds_exporter index 0dccbce9a196..3277cc1e59b8 160000 --- a/sources/rds_exporter/src/github.com/percona/rds_exporter +++ b/sources/rds_exporter/src/github.com/percona/rds_exporter @@ -1 +1 @@ -Subproject commit 0dccbce9a196f1416a09d90ef975e4216f72c5ee +Subproject commit 3277cc1e59b82813eefffb9b17d758f49cd26111 diff --git a/sources/watchtower/src/github.com/percona/watchtower b/sources/watchtower/src/github.com/percona/watchtower new file mode 160000 index 000000000000..9821ea2e3cfc --- /dev/null +++ b/sources/watchtower/src/github.com/percona/watchtower @@ -0,0 +1 @@ +Subproject commit 9821ea2e3cfcf620de03116f97010d0b632c4ac9