From 442258ab0fc2021409ea6932f9e5be32a4873c2f Mon Sep 17 00:00:00 2001 From: Robert Timm Date: Tue, 2 Jul 2024 13:31:06 +0000 Subject: [PATCH 01/12] chore: update variables env --- variables.env | 72 +++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/variables.env b/variables.env index ea33a9e4e..0d11925b6 100644 --- a/variables.env +++ b/variables.env @@ -4,14 +4,14 @@ # Version # ############################################################################## # Version of the repository. -VERSION=3.0.0 +VERSION=2.0.0 # ############################################################################## # Deploy version # ############################################################################## # The version of the deploy directory. -WBS_DEPLOY_VERSION=3.0.0 +WBS_DEPLOY_VERSION=2.0.0 # ############################################################################## @@ -20,8 +20,8 @@ WBS_DEPLOY_VERSION=3.0.0 # Update only patch versions for security releases. # Choose latest version for major releases. # https://releases.wikimedia.org/mediawiki/ -WBS_WIKIBASE_VERSION=3.0.0 -MEDIAWIKI_VERSION=1.42.1 +WBS_WIKIBASE_VERSION=2.0.0 +MEDIAWIKI_VERSION=1.41.2 # ############################################################################## @@ -135,36 +135,36 @@ DEBIAN_IMAGE_URL=debian:bookworm-slim # Versions in REL_ branches ensure compatibility with respective mediawiki versions. # Shouldn't require much of a review. # -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Wikibase/+/refs/heads/REL1_42 -WIKIBASE_COMMIT=1255c6e3ce8c14b72f0fa49ad98ba5fb388fbc0c -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Babel/+/refs/heads/REL1_42 -BABEL_COMMIT=d67b57379ae9d4ebb68ae764a6f9d05c8bf6c87d -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/cldr/+/refs/heads/REL1_42 -CLDR_COMMIT=6c28f1b99f9a7ea0eb7e11f48102805fd11a337d -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/CirrusSearch/+/refs/heads/REL1_42 -CIRRUSSEARCH_COMMIT=9cfe80151727a6950d278238f54db31aee889dd0 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/ConfirmEdit/+/refs/heads/REL1_42 -CONFIRMEDIT_COMMIT=48757725a81a500c5121517ea7c72c6926279718 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Elastica/+/refs/heads/REL1_42 -ELASTICA_COMMIT=382af148ab67640ca2ce213df245a1617487db68 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/EntitySchema/+/refs/heads/REL1_42 -ENTITYSCHEMA_COMMIT=7e66b541c9c0dec2caf316c9525334fbbf397ec5 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Nuke/+/refs/heads/REL1_42 -NUKE_COMMIT=54a2121e0855e445c32f3b9630e4410e53536bf6 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/OAuth/+/refs/heads/REL1_42 -OAUTH_COMMIT=fccfb680cc4bc9eae094f0356967e1b77faa88c9 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Scribunto/+/refs/heads/REL1_42 -SCRIBUNTO_COMMIT=f1d1e8adaac228e916cb6b7f6d6425037c928d64 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/SyntaxHighlight_GeSHi/+/refs/heads/REL1_42 -SYNTAXHIGHLIGHT_GESHI_COMMIT=e4fabbb7b4665e6066e931c5c48f39e0774c803a -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/UniversalLanguageSelector/+/refs/heads/REL1_42 -UNIVERSALLANGUAGESELECTOR_COMMIT=752ea5965b7b93f4e14fa861d587b0966b15413d -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/VisualEditor/+/refs/heads/REL1_42 -VISUALEDITOR_COMMIT=5c2c9b3a175daa5c6697d1241409311739d7a302 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseCirrusSearch/+/refs/heads/REL1_42 -WIKIBASECIRRUSSEARCH_COMMIT=0d8f5907ea9f4274e28ea2707440b176b2d8c071 -# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseManifest/+/refs/heads/REL1_42 -WIKIBASEMANIFEST_COMMIT=5413c72af830a031fbf485b9c6b9e49057ac88c3 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Wikibase/+/refs/heads/REL1_41 +WIKIBASE_COMMIT=75b6021e53fc9f026b6fc66491780de4c96f9cdf +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Babel/+/refs/heads/REL1_41 +BABEL_COMMIT=16b01dd431ed195549c42f1ccb6001f87e938b3d +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/cldr/+/refs/heads/REL1_41 +CLDR_COMMIT=0a6f222dbff1bbecb5778592929338300dafe6cf +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/CirrusSearch/+/refs/heads/REL1_41 +CIRRUSSEARCH_COMMIT=e13e36edbe82377b0024c2f047d7e7d7e87d431a +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/ConfirmEdit/+/refs/heads/REL1_41 +CONFIRMEDIT_COMMIT=172971224982f9394c8d5df75458e87b76be7601 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Elastica/+/refs/heads/REL1_41 +ELASTICA_COMMIT=2b55cd702a207f2870c600ae58045ca47c33b342 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/EntitySchema/+/refs/heads/REL1_41 +ENTITYSCHEMA_COMMIT=c69bc1feb5bb7c7c9d9f7151a6ec18a1b1fef98f +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Nuke/+/refs/heads/REL1_41 +NUKE_COMMIT=df734f785b005c156f1251387cf8071942ea275b +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/OAuth/+/refs/heads/REL1_41 +OAUTH_COMMIT=349ce85ff3b90a4948c0f946f9d926101dc9d5b5 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Scribunto/+/refs/heads/REL1_41 +SCRIBUNTO_COMMIT=cf723baff841d631a1572d295402dfbb68284b21 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/SyntaxHighlight_GeSHi/+/refs/heads/REL1_41 +SYNTAXHIGHLIGHT_GESHI_COMMIT=caadae39a8ef90b9822f5a80b589a495289528f3 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/UniversalLanguageSelector/+/refs/heads/REL1_41 +UNIVERSALLANGUAGESELECTOR_COMMIT=bffe4b8e57a82ae3572e216a8d899d0c1fbe0e68 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/VisualEditor/+/refs/heads/REL1_41 +VISUALEDITOR_COMMIT=3cad08181332f151afb75fc4a77c4cfa7c532e2d +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseCirrusSearch/+/refs/heads/REL1_41 +WIKIBASECIRRUSSEARCH_COMMIT=ecfc37e243737225e3e634c9c657f43fb9cec0fb +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseManifest/+/refs/heads/REL1_41 +WIKIBASEMANIFEST_COMMIT=f609133a8229e29011cea06dd9d02ad0f0e5c683 # ############################################################################## @@ -177,7 +177,7 @@ WIKIBASEMANIFEST_COMMIT=5413c72af830a031fbf485b9c6b9e49057ac88c3 # https://github.com/ProfessionalWiki/WikibaseLocalMedia/commits/master WIKIBASELOCALMEDIA_COMMIT=b2aac56b81c25cd04708f1019a833c81f074a1f2 # https://github.com/ProfessionalWiki/WikibaseEdtf/commits/master -WIKIBASEEDTF_COMMIT=6e8ebf2818de4dd43a3f39d290e46a1626db1b22 +WIKIBASEEDTF_COMMIT=5d5d59da21c50d12d1a3371c766480ff4126b7b4 # https://github.com/magnusmanske/quickstatements/commits/master QUICKSTATEMENTS_COMMIT=c4b2c6b086b319aa32dcdd7a323edf188faaa66d # https://bitbucket.org/magnusmanske/magnustools/commits/branch/master @@ -192,7 +192,7 @@ WBS_QUICKSTATEMENTS_VERSION=1.0.0 # No versioning scheme. Review changes carefully. # # https://gerrit.wikimedia.org/r/plugins/gitiles/wikidata/query/gui/+/refs/heads/master -WDQSQUERYGUI_COMMIT=b8fe9f1dbba7677f19d23b888e001e7c20978075 +WDQSQUERYGUI_COMMIT=7638030f312c21cc5a4fccd05e16d6ffffdd9220 # ############################################################################## From 8eab02a3a7e7a9e9066777d1e5d94f3b127b427c Mon Sep 17 00:00:00 2001 From: Robert Timm Date: Tue, 2 Jul 2024 13:31:41 +0000 Subject: [PATCH 02/12] chore: reference wikibase:2 in deploy --- deploy/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e8f9aaea5..8e5278631 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -6,7 +6,7 @@ services: # -------------------------------------------------- wikibase: - image: wikibase/wikibase:3 + image: wikibase/wikibase:2 depends_on: mysql: condition: service_healthy @@ -39,7 +39,7 @@ services: start_period: 5m wikibase-jobrunner: - image: wikibase/wikibase:3 + image: wikibase/wikibase:2 command: /jobrunner-entrypoint.sh depends_on: wikibase: From 0919af8f729baa13f99d4c08f1e6a9746ac80ed5 Mon Sep 17 00:00:00 2001 From: Robert Timm Date: Tue, 2 Jul 2024 13:32:49 +0000 Subject: [PATCH 03/12] docs: changes.md --- CHANGES.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0ff513e9d..e27cf43fc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,30 @@ This file is intended for use by the developers of this repository and of the re ## July 2024 +### Wikibase Release Pipeline 2.0.0 + +#### WBS Images + +| Image | WBS Version | Upstream Version | Base Image | Distribution | +| ------------------- | ----------- | -------------------- | ----------------------------- | --------------- | +| WBS Wikibase | 2.0.0 | MediaWiki 1.41.2 | PHP 8.3.8 apache | Debian Bookworm | +| WBS WDQS | 1.0.0 | WDQS 0.3.142 | JRE Eclipse Temurin 8u412-b08 | Debian Bookworm | +| WBS WDQS Frontend | 1.0.0 | N/A | Nginx 1.27.0 | Debian Bookworm | +| WBS WDQS Proxy | 1.0.0 | N/A | Nginx 1.27.0 | Debian Bookworm | +| WBS QuickStatements | 1.0.0 | N/A | PHP 8.3.8 Apache | Debian Bookworm | +| WBS ElasticSearch | 1.0.0 | ElasticSearch 7.10.2 | N/A | N/A | + +#### WBS Deploy 2.0.0 + +https://phabricator.wikimedia.org/T368254 + +On top of the images above, WBS Deploy uses: + +| Image | Upstream Version | +| ------------------- | -------------------- | +| MariaDB | 10.11 | +| Traefik | 3.1 | + ### Wikibase Release Pipeline 3.0.0 #### WBS Images From fe2130c0f368c0675bcc1745f2610c0eff79825b Mon Sep 17 00:00:00 2001 From: rti Date: Wed, 3 Jul 2024 13:18:39 +0200 Subject: [PATCH 04/12] chore: job runner restart rule (#711) --- deploy/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e8f9aaea5..05d4597a4 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -44,7 +44,7 @@ services: depends_on: wikibase: condition: service_healthy - restart: always + restart: unless-stopped volumes_from: - wikibase From 65fb7f7a7e1f6ff07da806fc538969ee4b533015 Mon Sep 17 00:00:00 2001 From: rti Date: Wed, 3 Jul 2024 13:18:39 +0200 Subject: [PATCH 05/12] chore: job runner restart rule (#711) --- deploy/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 8e5278631..d5e04472d 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -44,7 +44,7 @@ services: depends_on: wikibase: condition: service_healthy - restart: always + restart: unless-stopped volumes_from: - wikibase From b1f3b339bbddf9df47eda460a7360982cbd87ffb Mon Sep 17 00:00:00 2001 From: Loren Johnson Date: Wed, 3 Jul 2024 14:21:58 +0200 Subject: [PATCH 06/12] Make tests work across all releases (#712) --- test/package-lock.json | 7 ++++ test/package.json | 1 + .../extensions/universal-language-selector.ts | 40 +++++++++++-------- test/specs/repo/property.ts | 18 +++++++-- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/test/package-lock.json b/test/package-lock.json index 8c9a06866..a65a78c22 100644 --- a/test/package-lock.json +++ b/test/package-lock.json @@ -32,6 +32,7 @@ "npm-run-all": "^4.1.5", "prettier": "^3.2.5", "prettier-plugin-organize-imports": "^3.2.4", + "semver-parser": "^4.1.6", "stylelint-config-wikimedia": "0.16.1", "ts-node": "^10.9.2", "tslib": "^2.6.2", @@ -10676,6 +10677,12 @@ "node": ">=10" } }, + "node_modules/semver-parser": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/semver-parser/-/semver-parser-4.1.6.tgz", + "integrity": "sha512-i37y4KxCbaYXIwQKLLUC2nsFYxCeiguMrS6lfv4rU0aHd59fSAypcE+jGib2sbQRcJp2C2NL5ngKAqpKqd0SJw==", + "dev": true + }, "node_modules/semver/node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", diff --git a/test/package.json b/test/package.json index f02073850..340a3ee70 100644 --- a/test/package.json +++ b/test/package.json @@ -28,6 +28,7 @@ "npm-run-all": "^4.1.5", "prettier": "^3.2.5", "prettier-plugin-organize-imports": "^3.2.4", + "semver-parser": "^4.1.6", "stylelint-config-wikimedia": "0.16.1", "ts-node": "^10.9.2", "tslib": "^2.6.2", diff --git a/test/specs/repo/extensions/universal-language-selector.ts b/test/specs/repo/extensions/universal-language-selector.ts index 0384e5307..2d36aade3 100644 --- a/test/specs/repo/extensions/universal-language-selector.ts +++ b/test/specs/repo/extensions/universal-language-selector.ts @@ -1,3 +1,4 @@ +import { parseSemVer } from 'semver-parser'; import page from '../../../helpers/pages/page.js'; describe( 'UniversalLanguageSelector', function () { @@ -8,23 +9,30 @@ describe( 'UniversalLanguageSelector', function () { it( 'Should be able to see the language selector menu', async function () { await page.open( '' ); - await $( '#searchform input' ).click(); + // MediaWiki 1.39 default skin sets up language selector differently than subsequent versions, + // this exception can be removed once MW 1.39 is no longer supported. + if ( parseSemVer( testEnv.vars.MEDIAWIKI_VERSION ).minor === 39 ) { + await $( '#searchInput' ).click(); + await $( '.imeselector' ).click(); - // work around lang selector not showing up the first time - // blur the search bar - await $( '.page-Main_Page' ).click(); - // focus search bar again, lang selector should be there now - await $( '#searchform input' ).click(); + await expect( $( '.imeselector-menu h3' ) ).toHaveText( 'English' ); + } else { + await $( '#searchform input' ).click(); + // work around lang selector not showing up the first time + // blur the search bar + await $( '.page-Main_Page' ).click(); + // focus search bar again, lang selector should be there now + await $( '#searchform input' ).click(); + await $$( '.imeselector' ) + .filter( async ( selector ) => selector.isClickable() )[ 0 ] + .click(); - await $$( '.imeselector' ) - .filter( async ( selector ) => selector.isClickable() )[ 0 ] - .click(); - - // We need to use getHTML(). If an element isn't interactable - // getText() returns an empty string. - // https://webdriver.io/docs/api/element/getText/ - await expect( - $( 'div.imeselector-menu h3.ime-list-title' ).getHTML() - ).resolves.toMatch( /English/ ); + // We need to use getHTML(). If an element isn't interactable + // getText() returns an empty string. + // https://webdriver.io/docs/api/element/getText/ + await expect( + $( 'div.imeselector-menu h3.ime-list-title' ).getHTML() + ).resolves.toMatch( /English/ ); + } } ); } ); diff --git a/test/specs/repo/property.ts b/test/specs/repo/property.ts index b3d5f8088..27bf9f4be 100644 --- a/test/specs/repo/property.ts +++ b/test/specs/repo/property.ts @@ -1,3 +1,4 @@ +import { parseSemVer } from 'semver-parser'; import WikibaseApi from 'wdio-wikibase/wikibase.api.js'; import PropertyPage from '../../helpers/pages/entity/property.page.js'; import page from '../../helpers/pages/page.js'; @@ -89,7 +90,9 @@ describe( 'Property', function () { it( 'Should display the added properties on the "Recent changes" page', async function () { await browser.waitForJobs(); - await $( '.vector-main-menu-dropdown' ).click(); + if ( parseSemVer( testEnv.vars.MEDIAWIKI_VERSION ).minor > 39 ) { + await $( '.vector-main-menu-dropdown' ).click(); + } await $( '=Recent changes' ).click(); await expect( $( `=(${ propertyId })` ) ).toExist(); await expect( $( `=(${ stringPropertyId })` ) ).toExist(); @@ -121,7 +124,12 @@ describe( 'Property', function () { await page.open( '/wiki/Special:SetLabelDescriptionAliases/' ); await $( 'label=ID:' ).click(); await browser.keys( propertyId.split( '' ) ); - await $( 'span=Continue' ).click(); + + if ( parseSemVer( testEnv.vars.MEDIAWIKI_VERSION ).minor === 39 ) { + await $( 'span=Set label, description and aliases' ).click(); + } else { + await $( 'span=Continue' ).click(); + } await $( 'label=Label:' ).click(); await browser.keys( `${ dataType.name } Label`.split( '' ) ); @@ -132,7 +140,11 @@ describe( 'Property', function () { `${ dataType.name } Alias A|${ dataType.name } Alias B`.split( '' ) ); - await $( 'span=Save changes' ).click(); + if ( parseSemVer( testEnv.vars.MEDIAWIKI_VERSION ).minor === 39 ) { + await $( 'span=Set label, description and aliases' ).click(); + } else { + await $( 'span=Save changes' ).click(); + } await expect( $( `span.wikibase-labelview-text=${ dataType.name } Label` ) From a6a552d3f424d22ef8e53bdcf68501abc0316592 Mon Sep 17 00:00:00 2001 From: rti Date: Thu, 4 Jul 2024 12:47:25 +0200 Subject: [PATCH 07/12] T367024: release checklist (#714) * docs: release checklist * docs: typo * fix: bold --- README.md | 50 +++++++++++++------------------------------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index d9c3216aa..8c10ac61b 100644 --- a/README.md +++ b/README.md @@ -134,47 +134,23 @@ For more information on testing, see the [README](./test/README.md). ## Release process -WBS Deployment Kit and WBS Service Images are released using this repository. The process involves defining all upstream component versions to be used, building service images, testing all the images together and finally publishing them. +WBS Deploy and WBS Images are released using this repository. The process involves updating all upstream component versions to be used, building images, testing all the images together and finally publishing them. ### Release checklist Phabricator template ``` -[ ] **Pending issues as subtasks**. If any open tickets need to be resolved and/or related changes need to be included in the release, add them as subtasks of this release ticket. (If this release was triggered by a MediaWiki bugfix release, consider only including bug/security issue fixes, and avoid breaking changes.) - -[ ] **Create branches** - -- **For bugfix releases** - - **Create a release preparation branch.** The release branch itself should already exist, e.g., `mw-1.40`. Create a release preparation branch off the existing release branch. Choose a name for the release prep branch like `releaseprep-1.40.2`. (Don't name it `mw-*`! That's the naming convention for release branches.) - - **Backport changes from main only if absolutely necessary.** Bugfix releases should contain as few changes as possible. Make sure that the `variables.env` file is not changed by incoming changes from main. -- **For major releases** - - **Create a new release branch off main,** e.g., `mw-1.44`. This branch is now equivalent to main. It will hold all bugfix releases for that major version in the future. - - **Create a release preparation branch off that release branch.** e.g., `releaseprep-1.44.0`. This second branch is meant for preparing the first release on that release branch. - -[ ] **Update `variables.env`** on the release preparation branch. You can find further instructions in the [variables.env](https://github.com/wmde/wikibase-release-pipeline/blob/main/variables.env) file itself. - -[ ] **Update `CHANGES.md`** on the release preparation branch. Add a section following the example of previous releases: update the different values. The spec will eventually point to the release tag's `variables.env` once the release is published. - -[ ] **Create a release PR** that merges the release preparation branch, e.g., `releaseprep-1.44.0`, into the corresponding release branch (`mw-1.44` or equivalent). - -[ ] **CI should be green**. Tests may need adjustments in order to pass for the new version. Bugfix releases are likely to pass without any adjustments. - -[ ] **Do a sanity check by manually reviewing a running instance**. This can be done locally on your machine or by [deploying](https://docs.google.com/document/d/1BGxcqt9CHbb-8dfWjK-lZmNoNcD08urb23JqtgoVTeg/edit#heading=h.6a8ctlepqn5d) to the [test system](https://wikibase-product-testing.wmcloud.org). You can find prebuilt images from your PR on the [GitHub Container Registry](https://github.com/wmde/wikibase-release-pipeline/pkgs/container/wikibase%2Fwikibase) tagged with `dev-BRANCHNAME`, e.g., `dev-releaseprep-wmde.17`. This tag can be referenced to set up an instance running your PR version. - -[ ] **Get two reviews on the release preparation PR** so that it is technically ready to be merged. Merging will eventually trigger the release to Docker Hub. - -[ ] **Prepare communication** by creating a [release announcement](https://drive.google.com/drive/folders/1kHhKKwHlwq_P9x4j8-UnzV72yq0AYpsZ) using a template. - -[ ] **Coordinate with ComCom on timing the publication of the release**. Talk to SCoT (ComCom, technical writer) about this. - -[ ] **Publish the release** by merging the release preparation branch into the release branch. **ATTENTION: This will automatically push images to Docker Hub, thereby releasng the new version!** - -[ ] **Update the [Docker install instructions](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md)** to reflect changes in the latest version. If you were making a bugfix release for an older release, this can be skipped. - -[ ] **Make sure the communication is sent.** - -[ ] **Update the deploy directory.** ☠️☠️☠️ Currently, for this to happen, we need to merge another PR to the release branch. This PR's only change should be making the `deploy` directory reference the new version. For this PR's pipeline to properly run, before merging, you need to manually delete from the git repo the tag of the version you just released. Merging this deploy directory update PR is technically a re-release. Hopefully this weirdness will be fixed soon. - -[ ] **Merge back to main**. Now is the time to merge anything you want back to main on the release branch. +- [ ] **Pending issues as subtasks**. If any open tickets need to be resolved and/or related changes need to be included in the release, add them as subtasks of this release ticket. +- [ ] **To release breaking changes** as a new major version of WBS Deploy, create a new branch called `deploy-X`, where `X` is the new major version. +- [ ] **Create a release PR** with the following changes targeting the appropriate `deploy-X` release branch. + - [ ] **Update `variables.env`** by adjusting WBS versions and upstream versions. You can find further instructions in the [variables.env](https://github.com/wmde/wikibase-release-pipeline/blob/main/variables.env) file itself. + - [ ] **Update `CHANGES.md`** by adding a section following the example of previous releases. + - [ ] **CI should be green**. Tests may need adjustments in order to pass for the new version. Minor releases are likely to pass without any adjustments. Try re-running tests on failure, some specs could be flaky. +- [ ] **Do a sanity check by manually reviewing a running instance using your build**. This can be done locally on your machine or on a public server. You can find built images from your release PR on the [GitHub Container Registry](https://github.com/wmde/wikibase-release-pipeline/pkgs/container/wikibase%2Fwikibase) tagged with `dev-BRANCHNAME`, e.g., `dev-releaseprep`. This tag can be used to set up an instance running your release PR version. +- [ ] **Get two reviews on the release PR** so that it is ready to be merged. **Merging to `deploy-X` later will trigger the release to Docker Hub.** Do not merge yet! +- [ ] **Prepare communication** by creating a [release announcement](https://drive.google.com/drive/folders/1kHhKKwHlwq_P9x4j8-UnzV72yq0AYpsZ) using a template. +- [ ] **Coordinate with ComCom on timing the publication of the release**. Talk to SCoT (ComCom, technical writer) about this. +- [ ] **Publish the release** by merging the release branch into the `deploy-X` branch. **ATTENTION: This will automatically push images to Docker Hub!** +- [ ] **Merge back to main in a separate PR** from `deploy-X` to have adjustments to `CHANGES.md` and alike available on `main` too. Changes from `variables.env` should only be taken from a release of the latest version so that `main` always references the build of the latest components. You`re done. **Congratulations!** ``` From dfdc86833a76f51662cc17ba6ea891984a460a19 Mon Sep 17 00:00:00 2001 From: rti Date: Fri, 5 Jul 2024 14:09:45 +0200 Subject: [PATCH 08/12] Dockerhub docs (#705) * docs: add dockerhub readme file for wikibas * docs: add dockerhub readmes to all images * style: lint * fix: link --- build/Elasticsearch/dockerhub.md | 21 +++++++++++++++++++++ build/QuickStatements/dockerhub.md | 21 +++++++++++++++++++++ build/WDQS-frontend/dockerhub.md | 21 +++++++++++++++++++++ build/WDQS-proxy/dockerhub.md | 21 +++++++++++++++++++++ build/WDQS/dockerhub.md | 21 +++++++++++++++++++++ build/Wikibase/dockerhub.md | 21 +++++++++++++++++++++ 6 files changed, 126 insertions(+) create mode 100644 build/Elasticsearch/dockerhub.md create mode 100644 build/QuickStatements/dockerhub.md create mode 100644 build/WDQS-frontend/dockerhub.md create mode 100644 build/WDQS-proxy/dockerhub.md create mode 100644 build/WDQS/dockerhub.md create mode 100644 build/Wikibase/dockerhub.md diff --git a/build/Elasticsearch/dockerhub.md b/build/Elasticsearch/dockerhub.md new file mode 100644 index 000000000..1aa451ca7 --- /dev/null +++ b/build/Elasticsearch/dockerhub.md @@ -0,0 +1,21 @@ +# Wikibase Suite Elasticsearch Image + +Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in production, allowing you to maintain a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). + +If you want to host your own WBS instance, head over to the [WBS Deploy documentation](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md). + +# Documentation + +Version specific documentation for this image is hosted in our [git repository](https://github.com/wmde/wikibase-release-pipeline/). + +## Currently supported versions + +- July 2024 [3.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-3/build/Elasticsearch/README.md) +- July 2024 [2.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-2/build/Elasticsearch/README.md) +- July 2024 [1.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-1/build/Elasticsearch/README.md) + +## Legacy versions + +- 17 April 2024 [7.10.2-wmde.20](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.20/build/Elasticsearch/README.md) +- 17 April 2024 [7.10.2-wmde.19](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.19/build/Elasticsearch/README.md) +- 29 April 2024 [7.10.2-wmde.18](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.18/build/Elasticsearch/README.md) diff --git a/build/QuickStatements/dockerhub.md b/build/QuickStatements/dockerhub.md new file mode 100644 index 000000000..b72259ab4 --- /dev/null +++ b/build/QuickStatements/dockerhub.md @@ -0,0 +1,21 @@ +# Wikibase Suite QuickStatements Image + +Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in production, allowing you to maintain a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). + +If you want to host your own WBS instance, head over to the [WBS Deploy documentation](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md). + +# Documentation + +Version specific documentation for this image is hosted in our [git repository](https://github.com/wmde/wikibase-release-pipeline/). + +## Currently supported versions + +- July 2024 [3.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-3/build/QuickStatements/README.md) +- July 2024 [2.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-2/build/QuickStatements/README.md) +- July 2024 [1.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-1/build/QuickStatements/README.md) + +## Legacy versions + +- 17 April 2024 [wmde.20](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.20/build/QuickStatements/README.md) +- 17 April 2024 [wmde.19](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.19/build/QuickStatements/README.md) +- 29 April 2024 [wmde.18](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.18/build/QuickStatements/README.md) diff --git a/build/WDQS-frontend/dockerhub.md b/build/WDQS-frontend/dockerhub.md new file mode 100644 index 000000000..28636c62f --- /dev/null +++ b/build/WDQS-frontend/dockerhub.md @@ -0,0 +1,21 @@ +# Wikibase Suite WDQS-Frontend Image + +Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in production, allowing you to maintain a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). + +If you want to host your own WBS instance, head over to the [WBS Deploy documentation](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md). + +# Documentation + +Version specific documentation for this image is hosted in our [git repository](https://github.com/wmde/wikibase-release-pipeline/). + +## Currently supported versions + +- July 2024 [3.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-3/build/WDQS-frontend/README.md) +- July 2024 [2.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-2/build/WDQS-frontend/README.md) +- July 2024 [1.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-1/build/WDQS-frontend/README.md) + +## Legacy versions + +- 17 April 2024 [wmde.20](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.20/build/WDQS-frontend/README.md) +- 17 April 2024 [wmde.19](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.19/build/WDQS-frontend/README.md) +- 29 April 2024 [wmde.18](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.18/build/WDQS-frontend/README.md) diff --git a/build/WDQS-proxy/dockerhub.md b/build/WDQS-proxy/dockerhub.md new file mode 100644 index 000000000..f18f6bf97 --- /dev/null +++ b/build/WDQS-proxy/dockerhub.md @@ -0,0 +1,21 @@ +# Wikibase Suite WDQS-Proxy Image + +Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in production, allowing you to maintain a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). + +If you want to host your own WBS instance, head over to the [WBS Deploy documentation](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md). + +# Documentation + +Version specific documentation for this image is hosted in our [git repository](https://github.com/wmde/wikibase-release-pipeline/). + +## Currently supported versions + +- July 2024 [3.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-3/build/WDQS-proxy/README.md) +- July 2024 [2.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-2/build/WDQS-proxy/README.md) +- July 2024 [1.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-1/build/WDQS-proxy/README.md) + +## Legacy versions + +- 17 April 2024 [wmde.20](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.20/build/WDQS-proxy/README.md) +- 17 April 2024 [wmde.19](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.19/build/WDQS-proxy/README.md) +- 29 April 2024 [wmde.18](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.18/build/WDQS-proxy/README.md) diff --git a/build/WDQS/dockerhub.md b/build/WDQS/dockerhub.md new file mode 100644 index 000000000..7f906b50c --- /dev/null +++ b/build/WDQS/dockerhub.md @@ -0,0 +1,21 @@ +# Wikibase Suite WDQS Image + +Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in production, allowing you to maintain a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). + +If you want to host your own WBS instance, head over to the [WBS Deploy documentation](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md). + +# Documentation + +Version specific documentation for this image is hosted in our [git repository](https://github.com/wmde/wikibase-release-pipeline/). + +## Currently supported versions + +- July 2024 [3.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-3/build/WDQS/README.md) +- July 2024 [2.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-2/build/WDQS/README.md) +- July 2024 [1.x.x](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-1/build/WDQS/README.md) + +## Legacy versions + +- 17 April 2024 [0.3.137-wmde.20](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.20/build/WDQS/README.md) +- 17 April 2024 [0.3.137-wmde.19](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.19/build/WDQS/README.md) +- 29 April 2024 [0.3.135-wmde.18](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.18/build/WDQS/README.md) diff --git a/build/Wikibase/dockerhub.md b/build/Wikibase/dockerhub.md new file mode 100644 index 000000000..83f0ec016 --- /dev/null +++ b/build/Wikibase/dockerhub.md @@ -0,0 +1,21 @@ +# Wikibase Suite Wikibase Image + +Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in production, allowing you to maintain a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). + +If you want to host your own WBS instance, head over to the [WBS Deploy documentation](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md). + +# Documentation + +Version specific documentation for this image is hosted in our [git repository](https://github.com/wmde/wikibase-release-pipeline/). + +## Currently supported versions + +- July 2024 [3.x.x on MediaWiki 1.42](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-3/build/Wikibase/README.md) +- July 2024 [2.x.x on MediaWiki 1.41](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-2/build/Wikibase/README.md) +- July 2024 [1.x.x on MediaWiki 1.39](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-1/build/Wikibase/README.md) + +## Legacy versions + +- 17 April 2024 [1.41.1-wmde.20](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.20/build/Wikibase/README.md) +- 17 April 2024 [1.40.3-wmde.19](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.19/build/Wikibase/README.md) +- 29 April 2024 [1.39.7-wmde.18](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.18/build/Wikibase/README.md) From 04c07f407e903daa7a66041e1ff74914c53d7575 Mon Sep 17 00:00:00 2001 From: rti Date: Fri, 5 Jul 2024 14:10:12 +0200 Subject: [PATCH 09/12] feat: tag quickstatements version (#719) --- variables.env | 9 +++++++++ versions.inc.sh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/variables.env b/variables.env index ea33a9e4e..08a9fb602 100644 --- a/variables.env +++ b/variables.env @@ -178,11 +178,20 @@ WIKIBASEMANIFEST_COMMIT=5413c72af830a031fbf485b9c6b9e49057ac88c3 WIKIBASELOCALMEDIA_COMMIT=b2aac56b81c25cd04708f1019a833c81f074a1f2 # https://github.com/ProfessionalWiki/WikibaseEdtf/commits/master WIKIBASEEDTF_COMMIT=6e8ebf2818de4dd43a3f39d290e46a1626db1b22 + + +# ############################################################################## +# Quickstatments +# ############################################################################## +# Updated automatically by ./build.sh update_hashes # https://github.com/magnusmanske/quickstatements/commits/master QUICKSTATEMENTS_COMMIT=c4b2c6b086b319aa32dcdd7a323edf188faaa66d # https://bitbucket.org/magnusmanske/magnustools/commits/branch/master MAGNUSTOOLS_COMMIT=5b8cea412000072a2c8753023c11472a4ac11ef5 WBS_QUICKSTATEMENTS_VERSION=1.0.0 +# Quickstatements is currently considered version 2 +# https://www.wikidata.org/wiki/Help:QuickStatements#Running_QuickStatements +QUICKSTATEMENTS_VERSION=2 # ############################################################################## diff --git a/versions.inc.sh b/versions.inc.sh index 92829a108..65d73e5fd 100644 --- a/versions.inc.sh +++ b/versions.inc.sh @@ -51,6 +51,8 @@ function version_tags() { tags+=("${WBS_ELASTICSEARCH_VERSION}-es${ELASTICSEARCH_VERSION}") elif [[ "$image_name" == "wdqs" ]]; then tags+=("${WBS_WDQS_VERSION}-wdqs${WDQS_VERSION}") + elif [[ "$image_name" == "quickstatements" ]]; then + tags+=("${WBS_QUICKSTATEMENTS_VERSION}-qs${QUICKSTATEMENTS_VERSION}") fi printf "%s\n" "${tags[@]}" From eb608d6d5c3c68a63d40cb3a928a976c6dd62bfd Mon Sep 17 00:00:00 2001 From: Dan Shick Date: Tue, 9 Jul 2024 14:17:44 +0100 Subject: [PATCH 10/12] Proofreading of READMEs (#720) --- README.md | 22 +++++----- build/Elasticsearch/README.md | 8 ++-- build/QuickStatements/README.md | 44 +++++++------------- build/WDQS-frontend/README.md | 16 +++----- build/WDQS-proxy/README.md | 21 +++++----- build/WDQS/README.md | 25 +++++------- build/Wikibase/README.md | 72 ++++++++++++--------------------- deploy/README.md | 46 ++++++++++----------- deploy/config/README.md | 2 +- test/README.md | 20 ++++----- 10 files changed, 115 insertions(+), 161 deletions(-) diff --git a/README.md b/README.md index 8c10ac61b..085bcf2ad 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in product If you want to host your own WBS instance, head over to the [WBS Deploy documentation](./deploy/README.md). -If you're looking for individual WBS Images, head over to [hub.docker.com/u/wikibase](https://hub.docker.com/u/wikibase). +If you're looking for individual WBS images, head over to [hub.docker.com/u/wikibase](https://hub.docker.com/u/wikibase). > 🔧 This document is intended for people developing WBS. @@ -22,13 +22,13 @@ This repository contains the Wikibase Suite toolset used for: ### Build ``` -# Build all Wikibase Suite Images +# Build all Wikibase Suite images $ ./build.sh # Build only the MediaWiki/Wikibase containers $ ./build.sh wikibase -# Build the wdqs container without using Docker's cache +# Build the WDQS container without using Docker's cache $ ./build.sh --no-cache wdqs ``` @@ -58,7 +58,7 @@ $ cd deploy $ docker compose up --wait ``` -Find more details in the [WBS Deploy documentations](./deploy/README.md). +Find more details in the [WBS Deploy documentation](./deploy/README.md). ## Development setup @@ -70,11 +70,11 @@ $ git config core.hooksPath .githooks ## Testing -Tests are organized in suites, which can be found in `test/suites`. Each suite runs a series of specs (tests) found in the `test/specs` directory. Which specs run in each suite by default are specified in the `.config.ts` file in each suite directory under the `specs` key. +Tests are organized in suites, which can be found in `test/suites`. Each suite runs a series of specs (tests) found in the `test/specs` directory. Which specs run by default in each suite are specified in the `.config.ts` file in each suite directory under the `specs` key. All test suites are run against the most recently built local Docker images, those with the `:latest` tag, which are also selected when no tag is specified. The `deploy` test suite runs against the remote Docker images specified in the configuration in the `./deploy` directory. -You can run the tests in the docker container locally exactly as they are run in CI by using `test.sh`. +You can run the tests in the Docker container locally exactly as they are run in CI by using `test.sh`. ## Examples usage of `./test.sh`: @@ -85,10 +85,10 @@ You can run the tests in the docker container locally exactly as they are run in # Run all test suites ./test.sh all -# Only run a single suite (e.g. repo) +# Only run a single suite (e.g., repo) ./test.sh repo -# Only run a specific file within the setup for any test suite (e.g., repo and the babel extension) +# Only run a specific file within the setup for any test suite (e.g., repo and the Babel extension) ./test.sh repo --spec specs/repo/extensions/babel.ts ``` @@ -134,7 +134,7 @@ For more information on testing, see the [README](./test/README.md). ## Release process -WBS Deploy and WBS Images are released using this repository. The process involves updating all upstream component versions to be used, building images, testing all the images together and finally publishing them. +WBS Deploy and WBS images are released using this repository. The process involves updating all upstream component versions to be used, building images, testing all the images together and finally publishing them. ### Release checklist Phabricator template @@ -144,13 +144,13 @@ WBS Deploy and WBS Images are released using this repository. The process involv - [ ] **Create a release PR** with the following changes targeting the appropriate `deploy-X` release branch. - [ ] **Update `variables.env`** by adjusting WBS versions and upstream versions. You can find further instructions in the [variables.env](https://github.com/wmde/wikibase-release-pipeline/blob/main/variables.env) file itself. - [ ] **Update `CHANGES.md`** by adding a section following the example of previous releases. - - [ ] **CI should be green**. Tests may need adjustments in order to pass for the new version. Minor releases are likely to pass without any adjustments. Try re-running tests on failure, some specs could be flaky. + - [ ] **CI should be green**. Tests may need adjustments in order to pass for the new version. Minor releases are likely to pass without any adjustments. Try re-running tests on failure -- some specs may be flaky. - [ ] **Do a sanity check by manually reviewing a running instance using your build**. This can be done locally on your machine or on a public server. You can find built images from your release PR on the [GitHub Container Registry](https://github.com/wmde/wikibase-release-pipeline/pkgs/container/wikibase%2Fwikibase) tagged with `dev-BRANCHNAME`, e.g., `dev-releaseprep`. This tag can be used to set up an instance running your release PR version. - [ ] **Get two reviews on the release PR** so that it is ready to be merged. **Merging to `deploy-X` later will trigger the release to Docker Hub.** Do not merge yet! - [ ] **Prepare communication** by creating a [release announcement](https://drive.google.com/drive/folders/1kHhKKwHlwq_P9x4j8-UnzV72yq0AYpsZ) using a template. - [ ] **Coordinate with ComCom on timing the publication of the release**. Talk to SCoT (ComCom, technical writer) about this. - [ ] **Publish the release** by merging the release branch into the `deploy-X` branch. **ATTENTION: This will automatically push images to Docker Hub!** -- [ ] **Merge back to main in a separate PR** from `deploy-X` to have adjustments to `CHANGES.md` and alike available on `main` too. Changes from `variables.env` should only be taken from a release of the latest version so that `main` always references the build of the latest components. +- [ ] **Merge back to main in a separate PR** from `deploy-X` to have adjustments to `CHANGES.md` and the like available on `main` too. Changes from `variables.env` should only be taken from a release of the latest version so that `main` always references the build of the latest components. You`re done. **Congratulations!** ``` diff --git a/build/Elasticsearch/README.md b/build/Elasticsearch/README.md index 30ee2a5f8..4794197b2 100644 --- a/build/Elasticsearch/README.md +++ b/build/Elasticsearch/README.md @@ -18,14 +18,14 @@ In order to run Wikibase Elasticsearch, you need: ### MediaWiki/Wikibase instance -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we +We suggest using the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we run all our tests against. Follow the setup instructions over there to get it up and running. -Be sure to add the `ELASTICSEARCH_HOST` environment variable to you Wikibase container. +Be sure to add the `ELASTICSEARCH_HOST` environment variable to your Wikibase container. ## Example -You can use the following example Docker Compose to setup and run the image. Your Wikibase will be available on [http://localhost](http://localhost). +You can use the following example Docker Compose configuration to setup and run the image. Your Wikibase will be available on [http://localhost](http://localhost). ```yml services: @@ -106,7 +106,7 @@ Official releases of this image can be found on [Docker Hub wikibase/elasticsear ## Tags and Versioning -This Elasticsearch Image is using [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +This Elasticsearch image is using [semantic versioning](https://semver.org/spec/v2.0.0.html). We provide several tags that relate to the versioning semantics. diff --git a/build/QuickStatements/README.md b/build/QuickStatements/README.md index a78d389ca..9c082332b 100644 --- a/build/QuickStatements/README.md +++ b/build/QuickStatements/README.md @@ -1,10 +1,8 @@ # Wikibase Suite QuickStatements Image -[QuickStatements](https://github.com/magnusmanske/quickstatements) is a tool to -batch-edit [Wikibase](https://www.mediawiki.org/wiki/Wikibase). +[QuickStatements](https://github.com/magnusmanske/quickstatements) is a tool to batch-edit [Wikibase](https://www.mediawiki.org/wiki/Wikibase). -This image contains Quickstatements and the required Magnustools library. It is -ready to be hooked up to MediaWiki OAuth on a WBS Wikibase Image. +This image contains Quickstatements and the required Magnustools library. It is ready to be hooked up to MediaWiki OAuth on a WBS Wikibase image. > 💡 This image is part of Wikibase Suite (WBS). [WBS Deploy](https://github.com/wmde/wikibase-release-pipeline/deploy/README.md) provides everything you need to self-host a Wikibase instance out of the box. @@ -19,10 +17,9 @@ ready to be hooked up to MediaWiki OAuth on a WBS Wikibase Image. ### MediaWiki/Wikibase instance with -#### Official WBS Wikibase Image +#### Official WBS Wikibase image -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we -run all our tests against. Follow the setup instructions over there to get it up and running. +We suggest to use the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we run all our tests against. Follow the setup instructions over there to get it up and running. #### Other Wikibase instances @@ -36,24 +33,15 @@ php /var/www/html/extensions/OAuth/maintenance/createOAuthConsumer.php \ --grants createeditmovepage --grants editpage --grants highvolume --jsonOnSuccess ``` -You can pass the consumer and secret token you got from your Wikibase instance -to this container using the environment variables `OAUTH_CONSUMER_KEY` and -`OAUTH_CONSUMER_SECRET`. +You can pass the consumer and secret token you got from your Wikibase instance to this container using the environment variables `OAUTH_CONSUMER_KEY` and `OAUTH_CONSUMER_SECRET`. ### Reverse proxy -If QuickStatements and Wikibase are running on the same IP address, a reverse -proxy is required to route HTTP requests to Wikibase or QuickStatements -depending on the URL used to access them. See the [example](#Example) below for -a reverse proxy setup using [Traefik](https://doc.traefik.io/traefik/). +If QuickStatements and Wikibase are running on the same IP address, a reverse proxy is required to route HTTP requests to Wikibase or QuickStatements depending on the URL used to access them. See the [example](#Example) below for a reverse proxy setup using [Traefik](https://doc.traefik.io/traefik/). ### DNS resolution -In order to authorize QuickStatements against Wikibase via OAuth, both services -need to be accessible via DNS domain names, both, from within the docker -network as well as from the users browser. The easiest way to archive this is -by connecting both Wikibase and Quickstatements to the internet and let them -use public DNS domain names. +In order to authorize QuickStatements against Wikibase via OAuth, both services need to be accessible via DNS domain names, both from within the Docker network as well as from the user's browser. The easiest way to archive this is by connecting both Wikibase and Quickstatements to the internet and letting them use public DNS domain names. ### Environment variables @@ -75,7 +63,7 @@ Variables in **bold** are required. ## Example -An example how to run this image together with the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) using Docker Compose. +Here's an example of how to run this image together with the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) using Docker Compose. ```yml services: @@ -198,7 +186,7 @@ Official releases of this image can be found on [Docker Hub wikibase/quickstatem ## Tags and Versioning -This image uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +This image uses [semantic versioning](https://semver.org/spec/v2.0.0.html). We provide several tags that relate to the versioning semantics. @@ -211,23 +199,19 @@ We provide several tags that relate to the versioning semantics. ## Known Issues -Quickstatements' "Run in background" option is not supported by this image. +QuickStatements' "Run in background" option is not supported by this image. -Quickstatements' "Batches" require an database and are not supported by this image. +QuickStatements' "Batches" require an database and are not supported by this image. ## Troubleshooting -If you see an error such as `mw-oauth exception` when trying to log in, check -that you have passed the correct consumer token and secret token to -QuickStatements. +If you see an error such as `mw-oauth exception` when trying to log in, check that you have passed the correct consumer token and secret token to QuickStatements. -If you have changed the value of $wgSecretKey $wgOAuthSecretKey since you made -the consumer, you'll need to make another new consumer or reissue the secret -token for the old one. +If you have changed the value of `$wgSecretKey` or `$wgOAuthSecretKey` since you made the consumer, you'll need to make another new consumer or reissue the secret token for the old one. ## Internal filesystem layout -Hooking into the internal filesystem can be used to extend the functionality of this image. +Hooking into the internal filesystem can extend the functionality of this image. | Directory | Description | | ------------------------------------------- | ------------------------------ | diff --git a/build/WDQS-frontend/README.md b/build/WDQS-frontend/README.md index 5d8cedbdf..f20573e4e 100644 --- a/build/WDQS-frontend/README.md +++ b/build/WDQS-frontend/README.md @@ -17,23 +17,19 @@ In order to run WDQS Frontend, you need: ### MediaWiki/Wikibase instance -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we -run all our tests against. Follow the setup instructions over there to get it up and running. +We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we run all our tests against. Follow the setup instructions over there to get it up and running. ### WDQS as server -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wdqs). +We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs). ### WDQS as updater -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Checkout the documentation how to run it in updater mode. +We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Check out the [documentation](https://wikitech.wikimedia.org/wiki/Wikidata_Query_Service) to learn how to run it in updater mode. ### Reverse proxy -If QuickStatements and Wikibase are running on the same IP address, a reverse -proxy is required to route HTTP requests to Wikibase or QuickStatements -depending on the URL used to access them. See the [example](#Example) below for -a reverse proxy setup using [Traefik](https://doc.traefik.io/traefik/). +If QuickStatements and Wikibase are running on the same IP address, a reverse proxy is required to route HTTP requests to Wikibase or QuickStatements, depending on the URL used to access them. See the [example](#Example) below for a reverse proxy setup using [Traefik](https://doc.traefik.io/traefik/). ### Environment variables @@ -48,7 +44,7 @@ a reverse proxy setup using [Traefik](https://doc.traefik.io/traefik/). ## Example -An example how to run this image together with the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) and [WBS WDQS Image](https://hub.docker.com/r/wikibase/wdqs) behind a [Traefik](https://hub.docker.com/_/traefik) reverse proxy using Docker Compose. +Here's an example of how to run this image together with the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) and [WBS WDQS image](https://hub.docker.com/r/wikibase/wdqs) behind a [Traefik](https://hub.docker.com/_/traefik) reverse proxy using Docker Compose. ```yml services: @@ -191,7 +187,7 @@ Official releases of this image can be found on [Docker Hub wikibase/wdqs-fronte ## Tags and Versioning -This WDQS Frontend Image is using [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +This WDQS Frontend image is using [semantic versioning](https://semver.org/spec/v2.0.0.html). We provide several tags that relate to the versioning semantics. diff --git a/build/WDQS-proxy/README.md b/build/WDQS-proxy/README.md index 33e75bb09..b7507c6db 100644 --- a/build/WDQS-proxy/README.md +++ b/build/WDQS-proxy/README.md @@ -1,6 +1,6 @@ -# Wikibase Suite WDQS Proxy Image +# Wikibase Suite WDQS Proxy image -Proxy to put in front of the WBS WDQS Image enforcing READONLY requests, query timeouts and limits access to blazegraph sparql endpoints. +This is a proxy to put in front of the WBS WDQS image that enforces READONLY requests, implements query timeouts and limits access to Blazegraph SPARQL endpoints. > 💡 This image is part of Wikibase Suite (WBS). [WBS Deploy](https://github.com/wmde/wikibase-release-pipeline/deploy/README.md) provides everything you need to self-host a Wikibase instance out of the box. @@ -16,16 +16,15 @@ In order to run WDQS Proxy, you need: ### MediaWiki/Wikibase instance -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we -run all our tests against. Follow the setup instructions over there to get it up and running. +We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we run all our tests against. Follow the setup instructions over there to get it up and running. ### WDQS as server -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wdqs). +We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs). ### WDQS as updater -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Checkout the documentation how to run it in updater mode. +We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wdqs), the same as used for WDQS server. Check out the [documentation](https://wikitech.wikimedia.org/wiki/Wikidata_Query_Service) to learn how to run it in updater mode. ## Environment variables @@ -36,7 +35,7 @@ We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wdq ## Example -An example how to run this image together with the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) and [WBS WDQS Image](https://hub.docker.com/r/wikibase/wdqs) using Docker Compose. +Here's an example of how to run this image together with the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) and [WBS WDQS image](https://hub.docker.com/r/wikibase/wdqs) using Docker Compose. ```yml services: @@ -146,7 +145,7 @@ Official releases of this image can be found on [Docker Hub wikibase/wdqs-fronte ## Tags and Versioning -This WDQS Frontend Image is using [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +This WDQS Frontend image is using [semantic versioning](https://semver.org/spec/v2.0.0.html). We provide several tags that relate to the versioning semantics. @@ -161,7 +160,7 @@ We provide several tags that relate to the versioning semantics. ## Internal filesystem layout -Hooking into the internal filesystem can be used to extend the functionality of this image. +Hooking into the internal filesystem can extend the functionality of this image. | File | Description | | --------------------------------- | --------------------------------------------------------------------------------------------------------- | @@ -177,6 +176,4 @@ This image is built from this [Dockerfile](https://github.com/wmde/wikibase-rele This image is maintained by the Wikibase Suite Team at [Wikimedia Germany (WMDE)](https://wikimedia.de). -If you have questions not listed above or need help, use this [bug report -form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start -a conversation with the engineering team. +If you have questions not listed above or need help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team. diff --git a/build/WDQS/README.md b/build/WDQS/README.md index 4031d4143..3c00d20dd 100644 --- a/build/WDQS/README.md +++ b/build/WDQS/README.md @@ -1,4 +1,4 @@ -# Wikibase Suite Wikidata Query Service Image +# Wikibase Suite Wikidata Query Service image The [Wikidata Query Service (WDQS)](https://www.mediawiki.org/wiki/Wikidata_Query_Service) provides a way for tools to access Wikibase data, via a SPARQL API. It is based on [Blazegraph](https://github.com/blazegraph/database/wiki/Main_Page). @@ -17,16 +17,15 @@ In order to run WDQS, you need: ### MediaWiki/Wikibase instance -We suggest to use the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we -run all our tests against. Follow the setup instructions over there to get it up and running. +We suggest using the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) because this is the image we run all our tests against. Follow the setup instructions over there to get it up and running. ### WDQS as server -One instance of the image to execute the actual WDQS daemon started using `/runBlazegraph.sh`. +You'll need one instance of the image to execute the actual WDQS daemon started using `/runBlazegraph.sh`. ### WDQS as updater -One instance of the image to execute the updater started using `/runUpdate.sh`. This polls changes from Wikibase. +You'll need one instance of the image to execute the updater started using `/runUpdate.sh`. This polls changes from Wikibase. ### WDQS Proxy for public facing setups @@ -52,7 +51,7 @@ Variables in **bold** are required. ## Example -An example how to run this image together with the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) using Docker Compose. +Here's an example of how to run this image together with the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) using Docker Compose. ```yml services: @@ -161,7 +160,7 @@ Official releases of this image can be found on [Docker Hub wikibase/wdqs](https ## Tags and Versioning -This image uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +This image uses [semantic versioning](https://semver.org/spec/v2.0.0.html). We provide several tags that relate to the versioning semantics. @@ -178,7 +177,7 @@ We provide several tags that relate to the versioning semantics. When upgrading between WDQS versions, the data stored in `/wdqs/data` may not be compatible with the newer version. When testing the new image, if no data appears to have been loaded into the Query Service, you'll need to reload the data. -If all changes still appear in [RecentChanges], simply removing `/wdqs/data` and restarting the service should reload all data. +If all changes still appear in [RecentChanges], removing `/wdqs/data` and restarting the service should reload all data. However, [RecentChanges] are periodically purged of older entries, as determined by the MediaWiki configuration [\$wgRCMaxAge](https://www.mediawiki.org/wiki/Manual:$wgRCMaxAge). @@ -189,7 +188,7 @@ If you can't use [RecentChanges], you'll need to reload from an RDF dump: ## Internal filesystem layout -Hooking into the internal filesystem can be used to extend the functionality of this image. +Hooking into the internal filesystem can extend the functionality of this image. | File | Description | | ---------------------------- | ---------------------------------------------------------------------------------------------- | @@ -197,13 +196,13 @@ Hooking into the internal filesystem can be used to extend the functionality of | `/wdqs/RWStore.properties` | Properties for the service | | `/templates/mwservices.json` | Template for MediaWiki services (populated and placed into `/wdqs/mwservices.json` at runtime) | -## Known Issues +## Known issues ### Updater keeps restarting In some situations the WDQS Updater enters a restart loop. A workaround is to start the updater once with manual `--init` `--start` parameters for the current day. -In the docker compose example provided above, you could do: +In the Docker Compose example provided above, you might run: ```sh docker compose stop wdqs-updater @@ -219,6 +218,4 @@ This image is built from this [Dockerfile](https://github.com/wmde/wikibase-rele This image is maintained by the Wikibase Suite Team at [Wikimedia Germany (WMDE)](https://wikimedia.de). -If you have questions not listed above or need help, use this [bug report -form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start -a conversation with the engineering team. +If you have questions not listed above or need help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team. diff --git a/build/Wikibase/README.md b/build/Wikibase/README.md index 04915df80..8fa6a8676 100644 --- a/build/Wikibase/README.md +++ b/build/Wikibase/README.md @@ -1,12 +1,8 @@ # Wikibase Suite Wikibase Image -[Wikibase](https://www.mediawiki.org/wiki/Wikibase) is a MediaWiki extension -for working with versioned semi-structured data in a central repository. +[Wikibase](https://www.mediawiki.org/wiki/Wikibase) is a MediaWiki extension for working with versioned, semi-structured data in a central repository. -This image contains the Wikibase extension running on top of MediaWiki. -Besides Wikibase itself, several other extensions are bundled. The -MediaWiki application runs on top of PHP on an Apache web server on a Debian -base image. +This image contains the Wikibase extension running on top of MediaWiki. Besides Wikibase itself, several other extensions are bundled. The MediaWiki application runs on top of PHP on an Apache web server in a Debian base image. > 💡 This image is part of Wikibase Suite (WBS). [WBS Deploy](https://github.com/wmde/wikibase-release-pipeline/deploy/README.md) provides everything you need to self-host a Wikibase instance out of the box. @@ -15,17 +11,17 @@ base image. | [Babel](https://www.mediawiki.org/wiki/Extension:Babel) | Adds a parser function to inform other users about language proficiency and categorize users of the same levels and languages. | | [CLDR](https://www.mediawiki.org/wiki/Extension:CLDR) | Provides functions to localize the names of languages, countries, currencies, and time units based on their language code. | | [ConfirmEdit](https://www.mediawiki.org/wiki/Extension:ConfirmEdit) | Adds CAPTCHAs for page saves and other user actions. | -| [Elastica](https://www.mediawiki.org/wiki/Extension:Elastica), [CirrusSearch](https://www.mediawiki.org/wiki/Extension:CirrusSearch), [WikibaseCirrusSearch](https://www.mediawiki.org/wiki/Extension:WikibaseCirrusSearch) | ElasticSearch integration for MediaWiki and Wikibase. | +| [Elastica](https://www.mediawiki.org/wiki/Extension:Elastica), [CirrusSearch](https://www.mediawiki.org/wiki/Extension:CirrusSearch), [WikibaseCirrusSearch](https://www.mediawiki.org/wiki/Extension:WikibaseCirrusSearch) | Elasticsearch integration for MediaWiki and Wikibase. | | [EntitySchema](https://www.mediawiki.org/wiki/Extension:EntitySchema) | Allows to store Shape Expression Schemas on wiki pages. | | [Nuke](https://www.mediawiki.org/wiki/Extension:Nuke) | Gives sysops the ability to mass delete pages. | | [OAuth](https://www.mediawiki.org/wiki/Extension:OAuth) | Allow users to safely authorize another application ("consumer") to use the MediaWiki action API on their behalf. | | [Scribunto](https://www.mediawiki.org/wiki/Extension:Scribunto) | Provides a framework for embedding scripting languages into MediaWiki pages. | -| [SyntaxHighlight](https://www.mediawiki.org/wiki/Extension:SyntaxHighlight) | Allows source code to be syntax highlighted on wiki pages. | +| [SyntaxHighlight](https://www.mediawiki.org/wiki/Extension:SyntaxHighlight) | Allows source code to be syntax-highlighted on the wiki pages. | | [UniversalLanguageSelector](https://www.mediawiki.org/wiki/Extension:UniversalLanguageSelector) | Tool that allows users to select a language and configure its support in an easy way. | | [VisualEditor](https://www.mediawiki.org/wiki/Extension:VisualEditor) | Allows for editing pages as rich content. | | [WikibaseEdtf](https://github.com/ProfessionalWiki/WikibaseEdtf) | Adds support for the Extended Date/Time Format (EDTF) Specification via a new data type. | | [WikibaseLocalMedia](https://github.com/ProfessionalWiki/WikibaseLocalMedia) | Adds support for local media files to Wikibase via a new data type. | -| [WikibaseManifest](https://www.mediawiki.org/wiki/Extension:WikibaseManifest) | API provided metadata for structured data repository. | +| [WikibaseManifest](https://www.mediawiki.org/wiki/Extension:WikibaseManifest) | API-provided metadata for structured data repository. | ## Requirements @@ -38,27 +34,17 @@ In order to run Wikibase, you need: ### Database -The database MediaWiki will connect to and store all its data in. Technically, -MediaWiki supports multiple database engines. Though, MariaDB is the most -commonly used. This is also the only engine used to test the image before -release. +This is the database MediaWiki will connect to and store all its data in. Technically, MediaWiki supports multiple database engines, but MariaDB is the most commonly used. This is also the only engine used to test the image before release. -### Configuration Volume +### Configuration volume -MediaWiki will generate a `LocalSettings.php` file on first launch. This file -needs to be maintained by you, as you own it after generation. The Configuration -Volume is the place where this file will be stored. +MediaWiki will generate a `LocalSettings.php` file on first launch. Once this file has been generated, you own and control it. This file is stored in the configuration volume. ### Environment variables for initial settings -Those variables are only respected on first launch for generating MediaWikis -`LocalSettings.php` file. When launching the image with a `LocalSettings.php` -file present in the Configuration Volume, environment variables will not be -taken into account. +These variables are only respected on first launch in order to generate MediaWiki's `LocalSettings.php` file. When launching the image with a `LocalSettings.php` file present in the configuration volume, environment variables will not be taken into account. -Variables in **bold** are required on first launch without `LocalSettings.php` -in the Configuration Volume. The image will fail to start if one of those -variables does not have a value. Default values do not need to be overwritten. +Variables in **bold** are required on first launch without `LocalSettings.php` in the configuration volume. The image will fail to start if one of those variables does not have a value. Default values do not need to be overwritten. | Variable | Default | Description | | ---------------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | @@ -69,26 +55,22 @@ variables does not have a value. Default values do not need to be overwritten. | **`MW_ADMIN_NAME`** | undefined | Admin username to create on MediaWiki first install | | **`MW_ADMIN_PASS`** | undefined | Admin password to use for admin account on first install | | **`MW_ADMIN_EMAIL`** | undefined | Admin password to use for admin account on first install | -| **`MW_WG_SERVER`** | undefined | $wgServer to use for MediaWiki. A value matching how this site is accessed from the user's browser is required. | -| **`MW_WG_SITENAME`** | "wikibase" | $wgSitename to use for MediaWiki | -| **`MW_WG_LANGUAGE_CODE`** | "en" | $wgLanguageCode to use for MediaWiki | -| `ELASTICSEARCH_HOST` | undefined | Hostname to an Elasticsearch server with the Wikibase Extension installed, such as [wikibase/elasticsearch](https://hub.docker.com/r/wikibase/elasticsearch). Leave this undefined to disable ElasticSearch. | -| `ELASTICSEARCH_PORT` | 9200 | Port which Elasticsearch is available on | -| `QUICKSTATEMENTS_PUBLIC_URL` | undefined | Public URL of the Quickstatements server, such as [wikibase/quickstatements](https://hub.docker.com/r/wikibase/quickstatements). Leave undefined to disable QuickStatements functionality. | +| **`MW_WG_SERVER`** | undefined | `$wgServer` to use for MediaWiki. A value matching how this site is accessed from the user's browser is required. | +| **`MW_WG_SITENAME`** | "wikibase" | `$wgSitename` to use for MediaWiki | +| **`MW_WG_LANGUAGE_CODE`** | "en" | `$wgLanguageCode` to use for MediaWiki | +| `ELASTICSEARCH_HOST` | undefined | Hostname of an Elasticsearch server with the Wikibase extension installed, such as [wikibase/elasticsearch](https://hub.docker.com/r/wikibase/elasticsearch). Leave this undefined to disable Elasticsearch. | +| `ELASTICSEARCH_PORT` | 9200 | Port on which Elasticsearch is available | +| `QUICKSTATEMENTS_PUBLIC_URL` | undefined | Public URL of the QuickStatements server, such as [wikibase/quickstatements](https://hub.docker.com/r/wikibase/quickstatements). Leave undefined to disable QuickStatements functionality. | -### Job Runner +### Job runner -MediaWiki/Wikibase depends on [jobs being run in the background](https://www.mediawiki.org/wiki/Manual:Job_queue). -This can be either done on HTTP request, or by a dedicated Job Runner. The -default configuration of this image requires such an external Job Runner. +MediaWiki/Wikibase depends on [jobs being run in the background](https://www.mediawiki.org/wiki/Manual:Job_queue). This can be either done on HTTP request or by a dedicated job runner. The default configuration of this image requires an external job runner like this. -To setup an external Job Runner, use this image for a second container, -overwrite the command to `/jobrunner-entrypoint.sh` and share the same -Configuration Volume with it. +To set up an external job runner, use this image for a second container, overwrite the command to `/jobrunner-entrypoint.sh` and share the same configuration volume with it. ## Example -An example how to run this image together with the [WBS Wikibase Image](https://hub.docker.com/r/wikibase/wikibase) using Docker Compose. +Here's an example of how to run this image together with the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) using Docker Compose. ```yml services: @@ -152,9 +134,9 @@ volumes: Official releases of this image can be found on [Docker Hub wikibase/wikibase](https://hub.docker.com/r/wikibase/wikibase). -## Tags and Versioning +## Tags and versioning -This Wikibase Image is using [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +This Wikibase image is using [semantic versioning](https://semver.org/spec/v2.0.0.html). We provide several tags that relate to the versioning semantics. @@ -168,7 +150,7 @@ We provide several tags that relate to the versioning semantics. ## Internal filesystem layout -Hooking into the internal filesystem can be used to extend the functionality of this image. +Hooking into the internal filesystem can extend the functionality of this image. | Directory | Description | | ------------------------------- | -------------------------------------------------------------------------------------------------------------- | @@ -183,16 +165,14 @@ Hooking into the internal filesystem can be used to extend the functionality of | ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `/default-extra-install.sh` | Script for automatically creating Elasticsearch indices and creating OAuth consumer for QuickStatements | | `/extra-install.sh` | Optional script for custom functionality to be ran with MediaWiki install (when generating LocalSettings.php) | -| `/templates/LocalSettings.wbs.php` | Wikibase specific settings appended to the MediaWiki install generated `LocalSettings.php`. In particular this loads the Wikibase repo and client as well as all the other bundled extensions. | +| `/templates/LocalSettings.wbs.php` | Wikibase-specific settings appended to the MediaWiki install generated `LocalSettings.php`. Specifically, this loads the Wikibase repo and client as well as all the other bundled extensions. | ## Source This image is built from this [Dockerfile](https://github.com/wmde/wikibase-release-pipeline/blob/main/build/Wikibase/Dockerfile). -## Authors & Contact +## Authors & contact This image is maintained by the Wikibase Suite Team at [Wikimedia Germany (WMDE)](https://wikimedia.de). -If you have questions not listed above or need help, use this [bug report -form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start -a conversation with the engineering team. +If you have questions not listed above or need help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team. diff --git a/deploy/README.md b/deploy/README.md index e677065ff..e675d0b12 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,10 +1,10 @@ # Wikibase Suite Deploy -Wikibase Suite (WBS) Deploy is a containerized, production-ready [Wikibase](https://wikiba.se) system that allows you to self-host a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). In addition to Wikibase on MediaWiki, WBS Deploy includes the Wikidata Query Service (WDQS), QuickStatements, Elasticsearch, and a Traefik reverse proxy with SSL termination and ACME support. The service orchestration is implemented using docker-compose. +Wikibase Suite (WBS) Deploy is a containerized, production-ready [Wikibase](https://wikiba.se) system that allows you to self-host a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). In addition to Wikibase on MediaWiki, WBS Deploy includes the Wikidata Query Service (WDQS), QuickStatements, Elasticsearch, and a Traefik reverse proxy with SSL termination and ACME support. The service orchestration is implemented using Docker Compose. -> 🔧 This document is for people wanting to self-host the full Wikibase Suite using Wikibase Suite Deploy. If you are looking for individual WBS Images, head over to [hub.docker.com/u/wikibase](https://hub.docker.com/u/wikibase). +> 🔧 This document is for people wanting to self-host the full Wikibase Suite using Wikibase Suite Deploy. If you are looking for individual WBS images, head over to [hub.docker.com/u/wikibase](https://hub.docker.com/u/wikibase). -> 💡 This document presumes familiarity with basic Linux administration tasks and with Docker/docker-compose. +> 💡 This document presumes familiarity with basic Linux administration tasks and with Docker and Docker Compose. ## What's in the box? @@ -23,7 +23,7 @@ WBS Deploy consists of the following services: ## Quickstart -> 💡 If you want to run a quick test on a machine that has no public IP address (such as your local machine), check our [FAQ item](#can-i-host-wikibase-suite-locally) below. +> 💡 If you want to run a quick test on a machine that has no public IP address (such as your local machine), check our [FAQ entry](#can-i-host-wikibase-suite-locally) below. ### Requirements @@ -42,11 +42,11 @@ WBS Deploy consists of the following services: #### Domain names -You need three DNS records that resolve to your machine's IP address, one for each user facing service. +You need three DNS records that resolve to your machine's IP address, one for each user-facing service: -- Wikibase e.g. "wikibase.example.com" -- QueryService e.g. "query.example.com" -- QuickStatements e.g. "quickstatements.example.com" +- Wikibase, e.g., "wikibase.example.com" +- QueryService, e.g., "query.example.com" +- QuickStatements, e.g., "quickstatements.example.com" ### Initial setup @@ -80,7 +80,7 @@ docker compose up --wait The first start can take a couple of minutes. Wait for your shell prompt to return. -🎉 Congratulations, your Wikibase Suite instance should be up and running. Web interfaces are available over HTTPS (port 443) for the domain names you configured for Wikibase, the WDQS front end and Quickstatements. +🎉 Congratulations, your Wikibase Suite instance should now be up and running. Web interfaces are available over HTTPS (port 443) for the domain names you configured for Wikibase, the WDQS front end and Quickstatements. > 💡 If anything goes wrong, you can run `docker logs ` to see some hopefully helpful error messages. @@ -112,7 +112,7 @@ On first launch, WBS Deploy will create files in the `./config` directory alongs #### `config/LocalSettings.php` -This file is generated by the [MediaWiki installer script](https://www.mediawiki.org/wiki/Manual:Install.php) and supplemented by the Wikibase container's `entrypoint.sh` script on first launch. Once this file has been generated, you own and control it. This means that not only *can* you make changes, you may *need* to do so for [major version updates](https://www.mediawiki.org/wiki/Manual:Upgrading#Adapt_your_LocalSettings.php). +This file is generated by the [MediaWiki installer script](https://www.mediawiki.org/wiki/Manual:Install.php) and supplemented by the Wikibase container's `entrypoint.sh` script on first launch. Once this file has been generated, you own and control it. This means that not only *can* you make changes to it, you may *need* to do so for [major version updates](https://www.mediawiki.org/wiki/Manual:Upgrading#Adapt_your_LocalSettings.php). If `config/LocalSettings.php` is missing, it triggers the Wikibase container to run the [MediaWiki installer script](https://www.mediawiki.org/wiki/Manual:Install.php). If you need to run the installer again, you can remove the generated `LocalSettings.php` file (but keep a backup just in case!) and restart your instance. @@ -130,7 +130,7 @@ docker compose -f docker-compose.yml -f docker-compose.override.yml up --wait This way, your changes are kept separate from the original WBS Deploy code. ### Managing your data -Besides [your configuration](#configuring-your-wikibase-suite), it's your data that makes your instance unique. All instance data is stored in [Docker Volumes](https://docs.docker.com/storage/volumes/). +Besides [your configuration](#configuring-your-wikibase-suite), it's your data that makes your instance unique. All instance data is stored in [Docker volumes](https://docs.docker.com/storage/volumes/). - `wikibase-image-data`: MediaWiki image and media file uploads - `mysql-data`: MediaWiki/Wikibase MariaDB raw database @@ -140,7 +140,7 @@ Besides [your configuration](#configuring-your-wikibase-suite), it's your data t - `traefik-letsencrypt-data`: SSL certificates #### Back up your data -To back up your data, shut down the instance and dump the contents of all Docker Volumes into `.tar.gz` files. +To back up your data, shut down the instance and dump the contents of all Docker volumes into `.tar.gz` files. ```sh docker compose down @@ -158,7 +158,7 @@ done #### Restore from a backup -To restore the volume backups, ensure your instance has been shut down by running `docker compose down` and populate the Docker Volumes with data from your `.tar.gz` files. +To restore the volume backups, ensure your instance has been shut down by running `docker compose down` and populate the Docker volumes with data from your `.tar.gz` files. ```sh docker compose down @@ -178,9 +178,9 @@ done ### Updating and versioning -WBS uses [semantic versioning](https://semver.org/spec/v2.0.0.html). The WBS Deploy and all the WBS Images have individual version numbers. +WBS uses [semantic versioning](https://semver.org/spec/v2.0.0.html). The WBS Deploy and all the WBS images have individual version numbers. -WBS Deploy always references the latest minor and patch releases of the compatible WBS Images' major versions using the images' major version tag. +WBS Deploy always references the latest minor and patch releases of the compatible WBS images' major versions using the images' major version tag. #### Example @@ -188,11 +188,11 @@ Let's say the `wikibase` image version 1.0.0 is the initial version released wit This way, WBS Deploy can always reference the latest compatible version by using the major version tag. Nothing needs to be updated in WBS Deploy itself. If the `wikibase` image version gets bumped to 2.0.0, that indicates a breaking change; in this case the new image would not receive the `1` tag. Instead, a new version of WBS Deploy would be released (in this case 4.0.0) and this one would use a new major version tag called `2` to reference the Wikibase image. -WBS Deploy may also receive minor and patch updates, but, as noted above, they are not required to update related WBS Images. +WBS Deploy may also receive minor and patch updates, but, as noted above, they are not required to update related WBS images. -#### Minor and patch updates for WBS Images +#### Minor and patch updates for WBS images -Because WBS Deploy always references the latest minor and patch releases of compatible WBS Images, non-breaking changes (including security updates) are applied automatically when re-creating Docker containers. +Because WBS Deploy always references the latest minor and patch releases of compatible WBS images, non-breaking changes (including security updates) are applied automatically when re-creating Docker containers. This is always safe to do. Simply run: @@ -200,7 +200,7 @@ This is always safe to do. Simply run: docker compose down docker compose up --wait ``` -> 💡 In order to **prevent** new versions of WBS Images being pulled on container restart, stop your containers using `docker compose stop` instead of `docker compose down`, which will keep the current containers intact. Note: this stops security updates from being applied. It is generally recommended to use `docker compose down`, which removes the containers and allows updates to be applied. +> 💡 In order to **prevent** new versions of WBS images being pulled on container restart, stop your containers using `docker compose stop` instead of `docker compose down`, which will keep the current containers intact. **Note**: this stops security updates from being applied. It is generally recommended to use `docker compose down`, which removes the containers and allows updates to be applied. #### Minor and patch updates for WBS Deploy @@ -215,11 +215,11 @@ git pull ``` > 💡 If you have made changes to `docker-compose.yml`, commit them to a separate branch and merge them with upstream changes as you see fit. -> 💡 Each major version of WBS Deploy always references exactly one major version of each of the WBS Images. Thus, updating WBS Deploy minor and patch versions from a major version's git branch will never lead to breaking changes in WBS service images. +> 💡 Each major version of WBS Deploy always references exactly one major version of each of the WBS images. Thus, updating WBS Deploy minor and patch versions from a major version's git branch will never lead to breaking changes in WBS service images. #### Major upgrades -Major version upgrades are performed by updating WBS Deploy's major version. This is done by changing your git checkout to the new major version branch. This may reference new major versions of WBS Images or involve breaking changes. In turn, those may require additional steps as described below. +Major version upgrades are performed by updating WBS Deploy's major version. This is done by changing your git checkout to the new major version branch. This may reference new major versions of WBS images or involve breaking changes. In turn, those may require additional steps as described below. WBS only supports updating from one major version to the next version in sequence. In order to upgrade from 1.x.x to 3.x.x, you must first upgrade from 1.x.x to 2.x.x and then to 3.x.x. @@ -282,7 +282,7 @@ docker compose up --wait #### Automatic updates -At the moment, WBS Deploy does not support automatic updates. To automatically deploy minor and patch updates including security fixes to your WBS Images, [restart your instance](#minor-and-patch-updates-for-wbs-service-containers) on a regular basis with a systemd timer, cron job, or similar. +At the moment, WBS Deploy does not support automatic updates. To automatically deploy minor and patch updates including security fixes to your WBS images, [restart your instance](#minor-and-patch-updates-for-wbs-service-containers) on a regular basis with a systemd timer, cron job, or similar. #### Downgrades @@ -299,7 +299,7 @@ docker compose down --volumes rm config/LocalSettings.php ``` -Removing the `traefik-letsencrypt-data` volume will request a new certificate from LetsEncrypt on the next launch of your instance. Certificate generation on LetsEncrypt is [rate-limited](https://letsencrypt.org/docs/rate-limits/). Eventually, you may be blocked from generating new certificates **for multiple days**. To avoid that outcome, change to the LetsEncrypt staging server by appending the following line to the `traefik` `command` stanza of your `docker-compose.yml` file: +Removing the `traefik-letsencrypt-data` volume will request a new certificate from LetsEncrypt on the next launch of your instance. Certificate generation on LetsEncrypt is [rate-limited](https://letsencrypt.org/docs/rate-limits/); eventually you may be blocked from generating new certificates **for multiple days**. To avoid that outcome, change to the LetsEncrypt staging server by appending the following line to the `traefik` `command` stanza of your `docker-compose.yml` file: ```yml --certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory ``` diff --git a/deploy/config/README.md b/deploy/config/README.md index f4aaed98f..c24b3ed2e 100644 --- a/deploy/config/README.md +++ b/deploy/config/README.md @@ -1,5 +1,5 @@ # Wikibase Suite Deployment Kit Configuration -This directory will contain your configuration files after the first run of the Wikibase Suite Deployment Kit. Feel free to make adjustments as you see fit. Backup this folder regularly in order to restore your configuration later. +This directory will contain your configuration files after the first run of the Wikibase Suite Deployment Kit. Feel free to make adjustments as you see fit. Back up this folder regularly in order to restore your configuration later. Find information on how to run the Wikibase Suite Deployment Kit in the [top level README](../README.md). diff --git a/test/README.md b/test/README.md index efe17b18e..004bb2792 100644 --- a/test/README.md +++ b/test/README.md @@ -1,4 +1,4 @@ -# Test-cases +# Test cases ## Wikibase @@ -13,11 +13,11 @@ ## Wikibase & WDQS/WDQS-frontend -- [x] Create an item on Wikibase, and verify it can be found on Query Service (including the label etc) -- [x] Delete an item on Wikibase, and verify that the item isn’t there on the query service any more -- [x] Create an item with a statement and verify that querying using prefixes work correctly (wdt:P1 etc.. ) +- [x] Create an item on Wikibase and verify it can be found on Query Service (including the label, etc.) +- [x] Delete an item on Wikibase and verify that the item is no longer there on the query service +- [x] Create an item with a statement and verify that querying using prefixes works correctly (`wdt:P1`, etc.. ) -## Wikibase Client & Repo +## Wikibase client & repo - [x] Create an item on repo with sitelink to client - [x] Create an item on repo and verify that client can reference it using wikitext @@ -30,13 +30,13 @@ ## Scribunto - [x] Verify Lua module can be executed -- [x] Verify Lua module can be executed within time-limit +- [x] Verify Lua module can be executed within time limit -## Wikibase Federated properties (using wikidata.org as source wiki) +## Wikibase federated properties (using wikidata.org as source wiki) - [x] Create an item on Wikibase and add a statement using a federated property - - [x] verify it is available with the correct rdf through the Entity page - - [x] verify it is available through the WDQS-frontend with the correct prefixes + - [x] Verify it is available with the correct rdf through the Entity page + - [x] Verify it is available through the WDQS-frontend with the correct prefixes ## Elasticsearch @@ -50,7 +50,7 @@ The behavior of the tests can be modified with several environment variables. - `WIKIBASE_URL`: protocol, host name and port of the MediaWiki installation. Defaults to `http://127.0.0.1:8080` (Vagrant). - `MW_SCRIPT_PATH`: path to `index.php`, `api.php` etc. under `WIKIBASE_URL`. Defaults to `/w`. - `WIKIBASE_PROPERTY_STRING`, `WIKIBASE_PROPERTY_URL`, etc.: Property ID of a property with datatype `string`, `url`, etc. – if not set, a new property of this type will be created each time the tests are run. (This will fail unless anonymous users are allowed to create properties on the wiki, so setting `WIKIBASE_PROPERTY_STRING` correctly is recommended.) -- `HEADED_TESTS`: set to `true` to run tests in a headed browser. Follow the test execution on http://localhost:7900/?autoconnect=1&resize=scale +- `HEADED_TESTS`: set to `true` to run tests in a headed browser. Follow the test execution on http://localhost:7900/?autoconnect=1&resize=scale . ## Write more tests From 9e74f29daa612d20bd912e113c8c108fd5d8442a Mon Sep 17 00:00:00 2001 From: rti Date: Mon, 15 Jul 2024 11:10:14 +0200 Subject: [PATCH 11/12] T369358: product name in template.env (#723) * fix: product name in template.env * docs: fix remaining occurrences of "deployment kit" --- deploy/config/README.md | 6 +++--- deploy/template.env | 6 +++--- test/suites/deploy/deploy.conf.ts | 5 ++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/deploy/config/README.md b/deploy/config/README.md index c24b3ed2e..83bb3a161 100644 --- a/deploy/config/README.md +++ b/deploy/config/README.md @@ -1,5 +1,5 @@ -# Wikibase Suite Deployment Kit Configuration +# Wikibase Suite Deploy Configuration -This directory will contain your configuration files after the first run of the Wikibase Suite Deployment Kit. Feel free to make adjustments as you see fit. Back up this folder regularly in order to restore your configuration later. +This directory will contain your configuration files after the first run of Wikibase Suite Deploy. Feel free to make adjustments as you see fit. Back up this folder regularly in order to restore your configuration later. -Find information on how to run the Wikibase Suite Deployment Kit in the [top level README](../README.md). +Find information on how to run Wikibase Suite Deploy in the [top level README](../README.md). diff --git a/deploy/template.env b/deploy/template.env index 12ad65b49..43e71544b 100644 --- a/deploy/template.env +++ b/deploy/template.env @@ -1,8 +1,8 @@ # ############################################################################## -# Wikibase Suite Deployment Kit initial configuration +# Wikibase Suite Deploy - initial configuration # -# This file is a template for the .env file used by the Wikibase Suite -# Deployment Kit docker-compose setup. +# This file is a template for the .env file used by the Wikibase Suite Deploy +# docker-compose setup. # # WARNING: Do not add comments on the same line as env vars: in some # environments they will be included in the var! diff --git a/test/suites/deploy/deploy.conf.ts b/test/suites/deploy/deploy.conf.ts index fa1452e87..4186a6aaa 100755 --- a/test/suites/deploy/deploy.conf.ts +++ b/test/suites/deploy/deploy.conf.ts @@ -3,10 +3,9 @@ import TestEnv from '../../setup/test-env.js'; import wdioConfig from '../../setup/wdio.conf.js'; // TODO: The specs and setup in this suite are deprecated -// now that the entire test suite runs off the Deployment Kit -// configuration. +// now that the entire test suite runs off the Deploy configuration. // -// The user stories which reflect our current Deployment Kit design, +// The user stories which reflect our current Deploy design, // among other things, are likely to cover the now deprecated // Upgrade test suite. // From 6099f9cb4b9218c7cec1da32d01b731cc3885207 Mon Sep 17 00:00:00 2001 From: rti Date: Mon, 15 Jul 2024 12:47:23 +0200 Subject: [PATCH 12/12] docs(wdqs): fix updater reinit help (#725) --- build/WDQS/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/WDQS/README.md b/build/WDQS/README.md index 3c00d20dd..d8ddd435b 100644 --- a/build/WDQS/README.md +++ b/build/WDQS/README.md @@ -206,7 +206,7 @@ In the Docker Compose example provided above, you might run: ```sh docker compose stop wdqs-updater -docker compose run --rm wdqs-updater bash -c '/wdqs/runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --entityNamespaces "$WDQS_ENTITY_NAMESPACES" --init --start $(date +%Y%m%d000000)' +docker compose run --rm wdqs-updater bash '/wdqs/runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --entityNamespaces "$WDQS_ENTITY_NAMESPACES" --init --start $(date +%Y%m%d000000)' docker compose start wdqs-updater ```