From de37f261cf19f56c343ab20153e80e35b182ad97 Mon Sep 17 00:00:00 2001 From: antje-s <62277258+antje-s@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:35:16 +0100 Subject: [PATCH] Create codelists2ttl_toplevels.py (#110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create codelists2ttl_toplevels.py * Create upload_changes.py * Create generate-ttl.yml * Create generate-and-commit-ttl.yml * Update generate-ttl.yml * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Create codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Update generate-ttl.yml added "import os" * Update codelists2ttl_toplevels.py * Update generate-ttl.yml * Update codelists2ttl_toplevels.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Update generate-ttl.yml * Update codelists2ttl_toplevels.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Create publish-to-wmo-codes-testing.yml * Create codelists2ttl.py * Delete scripts/codelists2ttl_sub-discipline.py * Delete scripts/codelists2ttl_toplevels.py * Update generate-ttl.yml * Update codelists2ttl.py * Update codelists2ttl.py * Update codelists2ttl.py * Update codelists2ttl.py * add README * minor code updates * update script args, doc * update CI * updates * Codelists2ttl finalization (#115) * fix recursion in codelists2ttl.py * Fix typo * fix flake8 --------- Co-authored-by: Tom Kralidis Co-authored-by: Ján Osuský --- .github/workflows/generate-and-commit-ttl.yml | 41 +++ .github/workflows/generate-ttl.yml | 30 ++ .../publish-to-wmo-codes-testing.yml | 41 +++ scripts/README.md | 102 +++++++ scripts/codelists2ttl.py | 256 ++++++++++++++++++ scripts/upload_changes.py | 255 +++++++++++++++++ 6 files changed, 725 insertions(+) create mode 100644 .github/workflows/generate-and-commit-ttl.yml create mode 100644 .github/workflows/generate-ttl.yml create mode 100644 .github/workflows/publish-to-wmo-codes-testing.yml create mode 100644 scripts/README.md create mode 100644 scripts/codelists2ttl.py create mode 100644 scripts/upload_changes.py diff --git a/.github/workflows/generate-and-commit-ttl.yml b/.github/workflows/generate-and-commit-ttl.yml new file mode 100644 index 0000000..312e9ee --- /dev/null +++ b/.github/workflows/generate-and-commit-ttl.yml @@ -0,0 +1,41 @@ +name: Generate WIS2 Topic Hierarchy as TTL files and commit + +on: + push: + branches: + - main + paths: + - '**.yml' + - 'topic-hierarchy/**.csv' + +jobs: + main: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Generate TTL files + run: | + python3 scripts/codelists2ttl_toplevels.py + - name: checkout publication branch + uses: actions/checkout@v3 + with: + ref: publication + - name: update publication branch and publish + run: | + mkdir /tmp/wis/topic-hierarchy + mv wis/topic-hierarchy/ tmp/wis/ + git checkout publication + git config --global user.email "tomkralidis@gmail.com" + git config --global user.name "Tom Kralidis" + rm -rf wis/topic-hierarchy/* + cp -rpf /tmp/wis/topic-hierarchy/* . + git add . + git commit -am "update WIS2 Topic Hierarchy TTL files" + git push diff --git a/.github/workflows/generate-ttl.yml b/.github/workflows/generate-ttl.yml new file mode 100644 index 0000000..7b5cdd4 --- /dev/null +++ b/.github/workflows/generate-ttl.yml @@ -0,0 +1,30 @@ +name: Generate WIS2 Topic Hierarchy as TTL files + +on: + pull_request: + paths: + - 'topic-hierarchy/**.csv' + - '**.py' + - '**.yml' + +jobs: + main: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install flake8 + - name: Generate TTL files + run: | + python3 scripts/codelists2ttl.py + - name: run flake8 + run: flake8 scripts/ diff --git a/.github/workflows/publish-to-wmo-codes-testing.yml b/.github/workflows/publish-to-wmo-codes-testing.yml new file mode 100644 index 0000000..fff9c8d --- /dev/null +++ b/.github/workflows/publish-to-wmo-codes-testing.yml @@ -0,0 +1,41 @@ +name: Publish TTL files to WMO Codes Registry testing environment + +env: + WMO_CODES_TEST_USER_ID: ${{ secrets.WMO_CODES_TEST_USER_ID }} + WMO_CODES_TEST_API_KEY: ${{ secrets.WMO_CODES_TEST_API_KEY }} + +on: + push: + branches: + - publication + paths: + - 'wis/**.ttl' + +jobs: + main: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + steps: + - uses: actions/checkout@v2 + with: + ref: publication + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Copy TTL files + run: | + mkdir /tmp/wis + cp -rp wis/ /tmp/ + - uses: actions/checkout@v2 + with: + ref: main + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install -r scripts/requirements.txt + # - name: update publication branch and publish + # run: | + # python3 scripts/upload_changes.py {WMO_CODES_TEST_USER_ID} {WMO_CODES_TEST_API_KEY} test /tmp/wis --status experimental diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..0ac6a3c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,102 @@ +# WMO Codes Registry management + +## Overview + +The scripts in this directory are used to manage the WIS2 Topic Hierarchy (WTH) +publication to the WMO Codes Registry. + +## The WMO Codes Registry + +The [WMO Codes Registry](https://codes.wmo.int) is an authoritative service that +provides a number of registers defining controlled vocabularies used in various +WMO standards and systems. + +The service provides an API in support of automated workflow to manage codelist +registers. The API is available as follows: + +- https://ci.codes.wmo.int: testing +- https://codes.wmo.int: production + +API usage requires an account and credentials. Contact WMO Secretariat to be +provided access to the WMO Codes Registry API (a GitHub user id is required). + +Once you receive access, an API Key is required to manage resources on the registry. +To create an API Key, once logged into the registry, select _Admin / Create a temporary password (API Key)_, +and click _Create password_ to generate an API key. + +## Mapping from WTH to the WMO Code Registry + +The overall setup of WTH publication to the WMO Codes Registry works as follows: + +`wis` / CSV filename (without file extension) / CSV row `Name` + +where: + +- `wis` is the root `reg:Register` +- `topic-hierarchy` is attached to the `wis` register as a sub-register + - each WTH CSV file is a `reg:Register` itself, attached to the `topic-hierarchy` register as a sub-register + - each row in a WTH CSV file is a `skos:Concept` tied to its sub-register +- `topic-hierarchy/earth-system-discipline` is attached to the `wis/topic-hierarchy` register as a sub-register + - each WTH CSV `index.csv` file is a `reg:Register` itself, attached to the relevant `topic-hierarchy` directory / register as a sub-register + - each row in a WTH CSV file is a `skos:Concept` tied to its sub-register + +## Publication workflows + +Managing WTH publication to the WMO Codes Registry involves the following steps: + +- creating the `wis` register - needed only once, mentioned only for completeness +- generating TTL files from CSV +- publishing TTL files to the WMO Codes Registry + +### Creating the `wis` register + +Use the action/button "Create register" in the web UI of the WMO Codes Registry and fill following attributes: +- Notation +- Label +- Description + +### Generating TTLs + +To generate TTL files, from the root of the repository, run the following command: + +```bash +python3 scripts/codeslists2ttl.py +``` + +This will create all TTL files in a directory called `wis`. + +### Publishing TTLs + +To upload TTL files, from the root of the repository, run the following command: + +```bash +python3 scripts/upload_changes.py user_id --status +``` + +where: + +- `user_id` is your GitHub userid +- `password` is the API Key (see the [#overview](Overview) with instructions on how to generate an API Key +- `environment` is whether to upload change to the testing or production environment +- `output-directory` is the resulting directly where TTL outputs should published from, that is `wis` +- `status` is either `experimental` or `stable`, please note that this value cannot be changed on existing entries + +The script has a few more options, notably `-h` that displays help. + +Examples: + +```bash +# publish to test environment on https://ci.codes.wmo.int with experimental status +python3 scripts/upload_changes.py tomkralidis API_KEY test wis --status experimental + +# publish to test environment on https://ci.codes.wmo.int with stable status +python3 scripts/upload_changes.py tomkralidis API_KEY test wis --status stable + +# publish to production environment on https://codes.wmo.int with experimental status +python3 scripts/upload_changes.py tomkralidis API_KEY prod wis --status experimental + +# publish to production environment on https://codes.wmo.int with stable status +python3 scripts/upload_changes.py tomkralidis API_KEY prod wis --status stable +``` + +This will create/update all resources on the WMO Codes Registry. diff --git a/scripts/codelists2ttl.py b/scripts/codelists2ttl.py new file mode 100644 index 0000000..f483a5b --- /dev/null +++ b/scripts/codelists2ttl.py @@ -0,0 +1,256 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import argparse +import csv +from pathlib import Path +import shutil +from string import Template +import os + + +def gen_skos_register(subregisters: list) -> str: + """ + Generate SKOS Register TTL + + :param name: identifier of collection + :param description: label of collection + + :returns: `str` of SKOS Register TTL + """ + + REGISTER = ''' +@prefix skos: . +@prefix dct: . +@prefix ldp: . +@prefix reg: . +@prefix rdfs: . + + a reg:Register , ldp:Container ; + rdfs:label "WIS" ; + reg:notation "wis" ; + dct:description "WIS2 Topic Hierarchy"@en ; + reg:subregister ''' + + REGISTER += ' , '.join(subregisters) + ' ; \n' + + REGISTER += 'rdfs:member' + ' , '.join(subregisters) + + REGISTER += ' .' + + return REGISTER.strip() + + +def gen_skos_subregister(name: str, description: str, + source: str = None) -> str: + """ + Generate SKOS Sub-register TTL + + :param name: identifier of collection + :param description: label of collection + + :returns: `str` of SKOS Sub-register TTL + """ + + SUBREGISTER = ''' +@prefix skos: . +@prefix dct: . +@prefix ldp: . +@prefix reg: . +@prefix rdfs: . + +<$name> a reg:Register , skos:Collection , ldp:Container ; + ldp:hasMemberRelation skos:member ; + rdfs:label "$name" ; + dct:description "$description"''' + + template_vars = { + 'name': name, + 'description': description + } + + SUBREGISTER += ' .' + + return Template(SUBREGISTER).substitute(template_vars).strip() + + +def gen_skos_concept(name: str, description: str, source: str = None) -> str: + """ + Generate SKOS Concept TTL + + :param name: identifier of collection + :param description: label of collection + + :returns: `str` of SKOS Concept TTL + """ + + CONCEPT = ''' +@prefix skos: . +@prefix rdfs: . +@prefix dct: . + +<$name> a skos:Concept ; + rdfs:label "$name" ; + skos:notation "$name" ; + dct:description "$description"@en''' + + template_vars = { + 'name': name, + 'description': description + } + + CONCEPT += ' .' + + return Template(CONCEPT).substitute(template_vars).strip() + + +def write_ttl_file(ttl: str, ttl_base_path: Path, relative_path: Path, + verbose: bool = False) -> None: + """ + Write TTL to file + + :param ttl: `str` the TTL to be written to the file + :param ttl_base_path: the base path/directory for TTL files + :param relative_path: the relative path of this TTL file + + :returns: `None` + """ + + file_path = ttl_base_path / relative_path + if verbose: + print(f' Generating {relative_path}') + with file_path.open('w') as fh: + fh.write(ttl) + + +def process_subdomain_index(relative_path: Path, csv_base_path: Path, + ttl_base_path: Path, + verbose: bool = False) -> None: + """ + Processes recursively all index.csv files in csv_base_path/relative_path + and writes output to ttl_base_path/relative_path/ + + :param relative_path: relative path to start with + :param csv_base_path: base path where to look for CSV files + :param ttl_base_path: base path where store generated TTL files + + :returns: `None` + """ + + if verbose: + print(f' processing {relative_path}') + current_csv_dir = csv_base_path / relative_path + current_ttl_dir = ttl_base_path / relative_path + current_ttl_dir.mkdir() + index_file_path = current_csv_dir / 'index.csv' + with index_file_path.open() as fh_index: + csv_reader = csv.DictReader(fh_index) + for csv_record in csv_reader: + file_name = csv_record['Name'] + '.ttl' + csv_sub_dir = current_csv_dir / csv_record['Name'] + if os.path.exists(csv_sub_dir): + if verbose: + print(f' creating sub-register {file_name}') + ttl = gen_skos_subregister(csv_record['Name'], + csv_record['Name']) + write_ttl_file(ttl, ttl_base_path, relative_path / file_name) + # recursion + process_subdomain_index(relative_path / csv_record['Name'], + csv_base_path, ttl_base_path, verbose) + else: + if verbose: + print(f' creating concept {file_name}') + ttl = gen_skos_concept(csv_record['Name'], + csv_record['Description']) + write_ttl_file(ttl, ttl_base_path, relative_path / file_name) + + +parser = argparse.ArgumentParser() +parser.add_argument('-v', '--verbose', action='store_true', + help='Print more details') +args = parser.parse_args() + +ROOT_PATH = Path.cwd() +CSV_FILES_PATH = ROOT_PATH / 'topic-hierarchy' +TTL_FILES_PATH = ROOT_PATH / 'wis/topic-hierarchy' +COLLECTIONS = [] + +print('Generating WIS2 Topic Hierarchy TTL files') + +ttl_files_path = ROOT_PATH / 'wis/topic-hierarchy' +if ttl_files_path.exists(): + print(f'removed {ttl_files_path}') + shutil.rmtree(ttl_files_path) +ttl_files_path.mkdir(parents=True) + +root_table = ROOT_PATH / 'topic-hierarchy.csv' + + +root_ttl = ROOT_PATH / 'wis' / 'topic-hierarchy.ttl' +with root_ttl.open('w') as fh: + ttl = gen_skos_subregister('topic-hierarchy', 'WIS2 Topic Hierarchy') + fh.write(ttl) + + +with root_table.open() as fh: + subregisters = [] + reader = csv.DictReader(fh) + for row in reader: + ttl_files_path = ROOT_PATH / 'wis/topic-hierarchy' + subregister_url = 'http://codes.wmo.int/wis/topic-hierarchy' + subregisters.append(f'<{subregister_url}>') + register_ttl_dir = ttl_files_path + register_ttl_file = register_ttl_dir / f'{row["Name"]}.ttl' + if args.verbose: + print(f'Generating {register_ttl_file}') + + if row['Name'] != 'earth-system-discipline': + with register_ttl_file.open('w') as fh2: + ttl = gen_skos_subregister(row['Name'], row['Description']) + fh2.write(ttl) + + concept_csv_file = ROOT_PATH / f'topic-hierarchy/{row["Name"]}.csv' + + with concept_csv_file.open() as fh3: + reader2 = csv.DictReader(fh3) + for row2 in reader2: + concept_ttl_dir = register_ttl_dir / f'{row["Name"]}' + if not os.path.exists(concept_ttl_dir): + concept_ttl_dir.mkdir() + concept_ttl_file = concept_ttl_dir / f'{row2["Name"]}.ttl' + if args.verbose: + print(f'Generating {concept_ttl_file}') + with concept_ttl_file.open('w') as fh4: + ttl = gen_skos_concept(row2['Name'], + row2['Description']) + fh4.write(ttl) + else: + with register_ttl_file.open('w') as fh2: + ttl = gen_skos_subregister(row['Name'], row['Description']) + fh2.write(ttl) + +print('Level 1-7 completed') + +print('Generating Level 8+') +process_subdomain_index(Path('earth-system-discipline'), CSV_FILES_PATH, + TTL_FILES_PATH, args.verbose) + +print('Done') diff --git a/scripts/upload_changes.py b/scripts/upload_changes.py new file mode 100644 index 0000000..f2857ae --- /dev/null +++ b/scripts/upload_changes.py @@ -0,0 +1,255 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import argparse +from pathlib import Path +import requests + +HEADERS = { + 'Content-type': 'text/turtle; charset=UTF-8' +} + +PROD_REGISTRY = 'https://codes.wmo.int' +TEST_REGISTRY = 'https://ci.codes.wmo.int' + + +def authenticate(base_url: str, user_id: str, + password: str) -> requests.Session: + """ + Constructs authenticated session (with JSESSIONID cookie) + + :param base_url: base URL of registry API + :param user_id: GitHub User ID + :param password: password + + :returns: Session for further interaction upon successful login + """ + + url = f'{base_url}/system/security/apilogin' + print(f'Authenticating at {url}') + + session = requests.Session() + + data = { + 'userid': f'https://api.github.com/users/{user_id}', + 'password': password + } + + auth = session.post(url, data=data) + + if auth.status_code != 200: + raise ValueError('Authentication failed') + + return session + + +def post(session: requests.Session, url: str, payload: str, + dry_run: bool, verbose: bool, status: str) -> None: + """ + Posts new content to the intended parent register + + :param session: API session + :param url: URL of HTTP POST + :param payload: HTTP POST payload + :param dry_run: whether to run as a dry run (simulates request only) + :param verbose: whether to provide verbose output + :param status: publication status (experimental, stable) + + :returns: `None` + """ + + params = { + 'status': status + } + + if not dry_run: + if verbose: + print(f' Posting to: {url}') + print(f' headers: {HEADERS}') + print(f' params: {params}') + + res = session.post(url, headers=HEADERS, data=payload.encode('utf-8'), + params=params, stream=False) + + if res.status_code != 201: + print(f' POST failed with {res.status_code} {res.reason}: {res.content.decode("utf-8")}') # noqa + elif verbose: + print(f' POST succeeded with {res.status_code} {res.reason}') + else: + print(f' HTTP POST (dry run) to: {url}') + print(f' headers: {HEADERS}') + print(f' params: {params}') + + return + + +def put(session: requests.Session, url: str, payload: str, + dry_run: bool, verbose: bool, status: str) -> None: + """ + Updates definition of a register or entity. + + :param session: API session + :param url: URL of HTTP POST + :param payload: HTTP POST payload + :param dry_run: whether to run as a dry run (simulates request only) + :param verbose: whether to provide verbose output + :param status: publication status (experimental, stable) + + :returns: `None` + """ + + params = { + 'status': status, + 'non-member-properties': 'true' + } + + if not dry_run: + if verbose: + print(f' HTTP PUT to: {url}') + print(f' headers: {HEADERS}') + print(f' params: {params}') + + res = session.put(url, headers=HEADERS, data=payload.encode('utf-8'), + params=params) + + if res.status_code != 204: + print(f' PUT failed with {res.status_code} {res.reason}: {res.content.decode("utf-8")}') # noqa + elif verbose: + print(f' PUT succeeded with {res.status_code} {res.reason}') + else: + print(f' HTTP PUT (dry run) to {url}') + print(f' headers: {HEADERS}') + print(f' params: {params}') + + return + + +def upload(session: requests.Session, url: str, payload: str, + dry_run: bool, verbose: bool, status: str) -> None: + """ + PUTs or POSTs given data depending if it already exists or not + + :param session: API session + :param url: URL of HTTP POST + :param payload: HTTP POST payload + :param dry_run: whether to run as a dry run (simulates request only) + :param verbose: whether to provide verbose output + :param status: publication status (experimental, stable) + + :returns: `None` + """ + + # to check existence adjust the URL + url_to_check = url + '/' + if verbose: + print(f' Checking {url_to_check} - ', end=' ') + + response = session.get(url_to_check) + + if response.status_code == 200: + if verbose: + print('Existing entry, using PUT') + put(session, url, payload, dry_run, verbose, status) + elif response.status_code == 404: + if verbose: + print('New entry, using POST') + url = '/'.join(url.split('/')[:-1]) + post(session, url, payload, dry_run, verbose, status) + else: + raise ValueError( + f'Cannot upload to {url}: {response.status_code} {response.reason}: {response.content.decode("utf-8")}' # noqa + ) + + +def upload_file(session: requests.Session, url: str, filepath: Path, + dry_run: bool, verbose: bool, status: str) -> None: + """ + Uploads given TTL file to the registry + + :param session: API session + :param url: URL of HTTP POST + :param filepath: `pathlib.Path` of filepath + :param dry_run: whether to run as a dry run (simulates request only) + :param verbose: whether to provide verbose output + :param status: publication status (experimental, stable) + + :returns: `None` + """ + + with filepath.open(encoding='utf-8') as fh: + ttl_data = fh.read() + rel_id = filepath.parent / filepath.stem + if filepath.stem == 'wis': + rel_id = filepath.stem + url = f'{url}/{rel_id}' + + print(f'Uploading {filepath} to {url}') + upload(session, url, ttl_data, dry_run, verbose, status) + + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('user_id', help='GitHub User ID') + parser.add_argument( + 'password', + help=f'Password or token generated at {TEST_REGISTRY}/ui/temporary-password' # noqa + ) + parser.add_argument('mode', help='Mode: test or prod') + parser.add_argument('directory', + help='Name of the directory with TTL files to upload') + parser.add_argument( + '-n', + '--dry-run', + action='store_true', + help='Print what would be uploaded without actually sending anything' + ) + parser.add_argument('-v', '--verbose', + action='store_true', help='Print more details') + parser.add_argument('-s', '--status', default='experimental', + help='Status (experimental, stable)') + + args = parser.parse_args() + + REGISTRY = TEST_REGISTRY + + if not Path(args.directory).is_dir(): + raise ValueError(f'Directory {args.directory} does not exists.') + if args.mode not in ['test', 'prod']: + raise ValueError('Mode must be either "test" or "prod"') + if args.status not in ['stable', 'experimental']: + raise ValueError('Mode must be either "stable" or "experimental"') + if args.mode == 'prod': + REGISTRY = PROD_REGISTRY + + print(f'Running upload against {REGISTRY}') + + session = authenticate(REGISTRY, args.user_id, args.password) + + # cleanup if needed + # session.delete('https://ci.codes.wmo.int/wis') + + for filename in Path(args.directory).rglob('*.ttl'): + upload_file(session, REGISTRY, filename, args.dry_run, + args.verbose, args.status) + + print('Done')