Create codelists2ttl_toplevels.py (#110)

* Create codelists2ttl_toplevels.py * Create upload_changes.py * Create generate-ttl.yml * Create generate-and-commit-ttl.yml * Update generate-ttl.yml * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Create codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Update generate-ttl.yml added "import os" * Update codelists2ttl_toplevels.py * Update generate-ttl.yml * Update codelists2ttl_toplevels.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Update codelists2ttl_toplevels.py * Update generate-ttl.yml * Update codelists2ttl_toplevels.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_sub-discipline.py * Update codelists2ttl_toplevels.py * Create publish-to-wmo-codes-testing.yml * Create codelists2ttl.py * Delete scripts/codelists2ttl_sub-discipline.py * Delete scripts/codelists2ttl_toplevels.py * Update generate-ttl.yml * Update codelists2ttl.py * Update codelists2ttl.py * Update codelists2ttl.py * Update codelists2ttl.py * add README * minor code updates * update script args, doc * update CI * updates * Codelists2ttl finalization (#115) * fix recursion in codelists2ttl.py * Fix typo * fix flake8 --------- Co-authored-by: Tom Kralidis <tomkralidis@gmail.com> Co-authored-by: Ján Osuský <Jan.Osusky@iblsoft.com>
wmo-im · Mar 14, 2024 · de37f26 · de37f26
1 parent 7212238
commit de37f26
Show file tree

Hide file tree

Showing 6 changed files with 725 additions and 0 deletions.
diff --git a/.github/workflows/generate-and-commit-ttl.yml b/.github/workflows/generate-and-commit-ttl.yml
@@ -0,0 +1,41 @@
+name: Generate WIS2 Topic Hierarchy as TTL files and commit
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '**.yml'
+      - 'topic-hierarchy/**.csv'
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Generate TTL files
+        run: |
+          python3 scripts/codelists2ttl_toplevels.py
+      - name: checkout publication branch
+        uses: actions/checkout@v3
+        with:
+          ref: publication
+      - name: update publication branch and publish
+        run: |
+          mkdir /tmp/wis/topic-hierarchy
+          mv wis/topic-hierarchy/ tmp/wis/
+          git checkout publication
+          git config --global user.email "tomkralidis@gmail.com"
+          git config --global user.name "Tom Kralidis"
+          rm -rf wis/topic-hierarchy/*
+          cp -rpf /tmp/wis/topic-hierarchy/* .
+          git add .
+          git commit -am "update WIS2 Topic Hierarchy TTL files"
+          git push
diff --git a/.github/workflows/generate-ttl.yml b/.github/workflows/generate-ttl.yml
@@ -0,0 +1,30 @@
+name: Generate WIS2 Topic Hierarchy as TTL files
+
+on:
+  pull_request:
+    paths:
+      - 'topic-hierarchy/**.csv'
+      - '**.py'
+      - '**.yml'
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          pip3 install flake8
+      - name: Generate TTL files
+        run: |
+          python3 scripts/codelists2ttl.py
+      - name: run flake8
+        run: flake8 scripts/
diff --git a/.github/workflows/publish-to-wmo-codes-testing.yml b/.github/workflows/publish-to-wmo-codes-testing.yml
@@ -0,0 +1,41 @@
+name: Publish TTL files to WMO Codes Registry testing environment
+
+env:
+  WMO_CODES_TEST_USER_ID: ${{ secrets.WMO_CODES_TEST_USER_ID }}
+  WMO_CODES_TEST_API_KEY: ${{ secrets.WMO_CODES_TEST_API_KEY }}
+
+on:
+  push:
+    branches:
+      - publication
+    paths:
+      - 'wis/**.ttl'
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: publication 
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Copy TTL files
+        run: |
+          mkdir /tmp/wis
+          cp -rp wis/ /tmp/
+      - uses: actions/checkout@v2
+        with:
+          ref: main
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          pip3 install -r scripts/requirements.txt
+     # - name: update publication branch and publish
+     #   run: |
+     #     python3 scripts/upload_changes.py {WMO_CODES_TEST_USER_ID} {WMO_CODES_TEST_API_KEY} test /tmp/wis --status experimental
diff --git a/scripts/README.md b/scripts/README.md
@@ -0,0 +1,102 @@
+# WMO Codes Registry management
+
+## Overview
+
+The scripts in this directory are used to manage the WIS2 Topic Hierarchy (WTH)
+publication to the WMO Codes Registry.
+
+## The WMO Codes Registry
+
+The [WMO Codes Registry](https://codes.wmo.int) is an authoritative service that
+provides a number of registers defining controlled vocabularies used in various
+WMO standards and systems.
+
+The service provides an API in support of automated workflow to manage codelist
+registers. The API is available as follows:
+
+- https://ci.codes.wmo.int: testing
+- https://codes.wmo.int: production
+
+API usage requires an account and credentials. Contact WMO Secretariat to be
+provided access to the WMO Codes Registry API (a GitHub user id is required).
+
+Once you receive access, an API Key is required to manage resources on the registry.
+To create an API Key, once logged into the registry, select _Admin / Create a temporary password (API Key)_,
+and click _Create password_ to generate an API key.
+
+## Mapping from WTH to the WMO Code Registry
+
+The overall setup of WTH publication to the WMO Codes Registry works as follows:
+
+`wis` / CSV filename (without file extension) / CSV row `Name`
+
+where:
+
+- `wis` is the root `reg:Register`
+- `topic-hierarchy` is attached to the `wis` register as a sub-register
+  - each WTH CSV file is a `reg:Register` itself, attached to the `topic-hierarchy` register as a sub-register
+  - each row in a WTH CSV file is a `skos:Concept` tied to its sub-register
+- `topic-hierarchy/earth-system-discipline` is attached to the `wis/topic-hierarchy` register as a sub-register
+  - each WTH CSV `index.csv` file is a `reg:Register` itself, attached to the relevant `topic-hierarchy` directory / register as a sub-register
+  - each row in a WTH CSV file is a `skos:Concept` tied to its sub-register
+
+## Publication workflows
+
+Managing WTH publication to the WMO Codes Registry involves the following steps:
+
+- creating the `wis` register - needed only once, mentioned only for completeness
+- generating TTL files from CSV
+- publishing TTL files to the WMO Codes Registry
+
+### Creating the `wis` register
+
+Use the action/button "Create register" in the web UI of the WMO Codes Registry and fill following attributes:
+- Notation
+- Label
+- Description
+
+### Generating TTLs
+
+To generate TTL files, from the root of the repository, run the following command:
+
+```bash
+python3 scripts/codeslists2ttl.py
+```
+
+This will create all TTL files in a directory called `wis`.
+
+### Publishing TTLs
+
+To upload TTL files, from the root of the repository, run the following command:
+
+```bash
+python3 scripts/upload_changes.py user_id <password> <environment> <output-directory> --status <status>
+```
+
+where:
+
+- `user_id` is your GitHub userid
+- `password` is the API Key (see the [#overview](Overview) with instructions on how to generate an API Key
+- `environment` is whether to upload change to the testing or production environment
+- `output-directory` is the resulting directly where TTL outputs should published from, that is `wis`
+- `status` is either `experimental` or `stable`, please note that this value cannot be changed on existing entries
+
+The script has a few more options, notably `-h` that displays help.
+
+Examples:
+
+```bash
+# publish to test environment on https://ci.codes.wmo.int with experimental status
+python3 scripts/upload_changes.py tomkralidis API_KEY test wis --status experimental
+
+# publish to test environment on https://ci.codes.wmo.int with stable status
+python3 scripts/upload_changes.py tomkralidis API_KEY test wis --status stable
+
+# publish to production environment on https://codes.wmo.int with experimental status
+python3 scripts/upload_changes.py tomkralidis API_KEY prod wis --status experimental
+
+# publish to production environment on https://codes.wmo.int with stable status
+python3 scripts/upload_changes.py tomkralidis API_KEY prod wis --status stable
+```
+
+This will create/update all resources on the WMO Codes Registry.