From f96c4033e9edf3df0d38c396d1c9d7cdcf239b1d Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Wed, 17 Jan 2024 14:45:14 -0800 Subject: [PATCH 01/14] chore: remove superfluous 1min wait --- proposals/16:upgrade-8/use.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/proposals/16:upgrade-8/use.sh b/proposals/16:upgrade-8/use.sh index 94a0d99d..fcff3a72 100644 --- a/proposals/16:upgrade-8/use.sh +++ b/proposals/16:upgrade-8/use.sh @@ -91,10 +91,6 @@ for i in "${govaccounts[@]}"; do agops perf satisfaction --from $i --executeOffer $VOTE_OFFER --keyring-backend=test done -## wait for the election to be resolved (1m default in commands/psm.js) -echo "waiting 1 minute for election to be resolved" -sleep 65 - echo DEBUG print mint limit agops psm info --pair ${PSM_PAIR} From cb090a921a73b3188d5ec986935a461954965c48 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 18 Jan 2024 10:43:56 -0800 Subject: [PATCH 02/14] docs: remove extraneous FIXME --- proposals/34:upgrade-10/performActions.js | 1 - proposals/43:upgrade-11/performActions.js | 1 - 2 files changed, 2 deletions(-) diff --git a/proposals/34:upgrade-10/performActions.js b/proposals/34:upgrade-10/performActions.js index 8d61564e..44306acf 100644 --- a/proposals/34:upgrade-10/performActions.js +++ b/proposals/34:upgrade-10/performActions.js @@ -1,6 +1,5 @@ #!/usr/bin/env node -// FIXME get TypeScript to resolve these, probably with lib:ES2022 import assert from 'node:assert/strict'; import { provisionWallet, diff --git a/proposals/43:upgrade-11/performActions.js b/proposals/43:upgrade-11/performActions.js index bc838c12..68b4ccb2 100644 --- a/proposals/43:upgrade-11/performActions.js +++ b/proposals/43:upgrade-11/performActions.js @@ -1,6 +1,5 @@ #!/usr/bin/env node -// FIXME get TypeScript to resolve these, probably with lib:ES2022 import assert from 'node:assert/strict'; import { agoric, agops } from '@agoric/synthetic-chain/src/lib/cliHelper.js'; From 9070132edaf26d89aea6baa0abb6f3a250c7c457 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 18 Jan 2024 10:45:06 -0800 Subject: [PATCH 03/14] chore: remove superfluous use.sh --- proposals/b:zoe1/use.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 proposals/b:zoe1/use.sh diff --git a/proposals/b:zoe1/use.sh b/proposals/b:zoe1/use.sh deleted file mode 100644 index cffcf2a8..00000000 --- a/proposals/b:zoe1/use.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Exit when any command fails -set -e - -echo "TODO remove once https://github.com/Agoric/agoric-3-proposals/pull/20 lands" From f0772db8d989c25162a876db3b207dca940f6a87 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 16 Jan 2024 08:43:17 -0800 Subject: [PATCH 04/14] refactor: genericize buildProposalImages --- packages/synthetic-chain/cli.ts | 4 ++-- packages/synthetic-chain/src/cli/build.ts | 8 ++++++-- packages/synthetic-chain/src/cli/proposals.ts | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index 7be4f6dc..cf40491b 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { execSync } from 'node:child_process'; import { buildProposalSubmissions, - buildTestImages, + buildProposalImages, readBuildConfig, } from './src/cli/build.js'; import { writeDockerfile } from './src/cli/dockerfileGen.js'; @@ -48,7 +48,7 @@ const buildImages = () => { 'cp -r node_modules/@agoric/synthetic-chain/upgrade-test-scripts .', ); buildProposalSubmissions(proposals); - buildTestImages(proposals, values.dry); + buildProposalImages(proposals, 'test', values.dry); }; switch (cmd) { diff --git a/packages/synthetic-chain/src/cli/build.ts b/packages/synthetic-chain/src/cli/build.ts index 74f84a0a..b7459969 100755 --- a/packages/synthetic-chain/src/cli/build.ts +++ b/packages/synthetic-chain/src/cli/build.ts @@ -62,14 +62,18 @@ export const buildProposalSubmissions = (proposals: ProposalInfo[]) => { } }; -export const buildTestImages = (proposals: ProposalInfo[], dry = false) => { +export const buildProposalImages = ( + proposals: ProposalInfo[], + stage: 'test' | 'use', + dry = false, +) => { for (const proposal of proposals) { if (!dry) { console.log( `\nBuilding test image for proposal ${proposal.proposalName}`, ); } - const { name, target } = imageNameForProposal(proposal, 'test'); + const { name, target } = imageNameForProposal(proposal, stage); // 'load' to ensure the images are output to the Docker client. Seems to be necessary // for the CI docker/build-push-action to re-use the cached stages. const cmd = `docker buildx build --load --tag ${name} --target ${target} .`; diff --git a/packages/synthetic-chain/src/cli/proposals.ts b/packages/synthetic-chain/src/cli/proposals.ts index 837849dd..b15c06f5 100644 --- a/packages/synthetic-chain/src/cli/proposals.ts +++ b/packages/synthetic-chain/src/cli/proposals.ts @@ -76,7 +76,7 @@ export function lastPassedProposal( export function imageNameForProposal( proposal: ProposalCommon, - stage: 'test' | 'eval', + stage: 'test' | 'use', ) { const target = `${stage}-${proposal.proposalName}`; return { From 061a127810a3aa6b58ce3a35045bdc571fe3a276 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 16 Jan 2024 08:48:21 -0800 Subject: [PATCH 05/14] refactor: rm extra cp start_agd --- packages/synthetic-chain/src/cli/dockerfileGen.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/synthetic-chain/src/cli/dockerfileGen.ts b/packages/synthetic-chain/src/cli/dockerfileGen.ts index 79033878..2ccad1ee 100755 --- a/packages/synthetic-chain/src/cli/dockerfileGen.ts +++ b/packages/synthetic-chain/src/cli/dockerfileGen.ts @@ -174,8 +174,6 @@ ENTRYPOINT ./run_test.sh ${proposalIdentifier}:${proposalName} # DEFAULT FROM use-${lastProposal.proposalName} -COPY --link --chmod=755 ./upgrade-test-scripts/start_agd.sh /usr/src/upgrade-test-scripts/ - WORKDIR /usr/src/upgrade-test-scripts SHELL ["/bin/bash", "-c"] ENTRYPOINT ./start_agd.sh From a7b9ac857fee2b84f0ab168b49f61476e0e0baf3 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 16 Jan 2024 09:10:55 -0800 Subject: [PATCH 06/14] feat: build and publish 'use' images --- README.md | 4 ++++ packages/synthetic-chain/cli.ts | 2 ++ packages/synthetic-chain/src/cli/proposals.ts | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 05c8a639..1ce5f804 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,10 @@ All proposals then have two additional stages: The `TEST` stage does not RUN as part of the build. It only defines the ENTRYPOINT and CI runs them all. +The `USE` stage is built into images that are pushed to the image repository. These can be used by release branches to source a particular state of the synthetic chain. + +Finally there is a `DEFAULT` target which take the last `USE` image and sets its entrypoing to `./start_agd.sh` which runs the chain indefinitely. This makes it easy to source that image as a way to _run_ the synthetic chain with all passed proposals. + ## Proposals ### Types diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index cf40491b..6e545d3b 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -48,6 +48,8 @@ const buildImages = () => { 'cp -r node_modules/@agoric/synthetic-chain/upgrade-test-scripts .', ); buildProposalSubmissions(proposals); + // the 'test' images need the 'use' images + buildProposalImages(proposals, 'use', values.dry); buildProposalImages(proposals, 'test', values.dry); }; diff --git a/packages/synthetic-chain/src/cli/proposals.ts b/packages/synthetic-chain/src/cli/proposals.ts index b15c06f5..246b5dcf 100644 --- a/packages/synthetic-chain/src/cli/proposals.ts +++ b/packages/synthetic-chain/src/cli/proposals.ts @@ -75,7 +75,7 @@ export function lastPassedProposal( } export function imageNameForProposal( - proposal: ProposalCommon, + proposal: Pick, stage: 'test' | 'use', ) { const target = `${stage}-${proposal.proposalName}`; From 47cee2749312c61e6975590b09e1a19293c983c0 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 16 Jan 2024 11:02:01 -0800 Subject: [PATCH 07/14] refactor: build test images in 'test' command --- .github/workflows/ci.yml | 4 ++-- packages/synthetic-chain/cli.ts | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fc795bd..58a94b99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,11 +113,11 @@ jobs: - run: corepack enable || sudo corepack enable - run: yarn install - - name: build test images ${{ matrix.platform }} == ${{ env.DEFAULT_PLATFORM }} + - name: build proposal images ${{ matrix.platform }} == ${{ env.DEFAULT_PLATFORM }} run: | docker info node_modules/.bin/synthetic-chain build ${{ matrix.platform == env.DEFAULT_PLATFORM && ' ' || '--dry' }} - - name: run test images + - name: run proposal tests if: ${{ matrix.platform == env.DEFAULT_PLATFORM }} run: node_modules/.bin/synthetic-chain test diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index 6e545d3b..6940d676 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -35,32 +35,33 @@ const [cmd] = positionals; // TODO consider a lib like Commander for auto-gen help const usage = `USAGE: -build - build the synthetic-chain +build - build the synthetic-chain "use" images -test [--debug] - run each proposal's test image +test [--debug] - build the "test" images and run them doctor - diagnostics and quick fixes `; -const buildImages = () => { +const buildUseImages = () => { execSync( // XXX very brittle 'cp -r node_modules/@agoric/synthetic-chain/upgrade-test-scripts .', ); buildProposalSubmissions(proposals); - // the 'test' images need the 'use' images buildProposalImages(proposals, 'use', values.dry); - buildProposalImages(proposals, 'test', values.dry); }; switch (cmd) { case 'build': { const { fromTag } = buildConfig; writeDockerfile(allProposals, fromTag); - buildImages(); + buildUseImages(); break; } case 'test': + // always rebuild all test images. Keeps it simple and these are fast + // as long as the "use" stages are cached because they don't execute anything themselves. + buildProposalImages(proposals, 'test', values.dry); if (values.debug) { debugTestImage(matchOneProposal(proposals, match!)); } else { From e5de5d2bb6755330fdf1a36c17d517b267b7e001 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 16 Jan 2024 11:13:42 -0800 Subject: [PATCH 08/14] ci: build 'use' images multiarch --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58a94b99..223fd87f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ concurrency: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + DEFAULT_PLATFORM: linux/amd64 jobs: platforms: @@ -113,10 +114,10 @@ jobs: - run: corepack enable || sudo corepack enable - run: yarn install - - name: build proposal images ${{ matrix.platform }} == ${{ env.DEFAULT_PLATFORM }} + - name: build proposal images run: | docker info - node_modules/.bin/synthetic-chain build ${{ matrix.platform == env.DEFAULT_PLATFORM && ' ' || '--dry' }} + node_modules/.bin/synthetic-chain build - name: run proposal tests if: ${{ matrix.platform == env.DEFAULT_PLATFORM }} run: node_modules/.bin/synthetic-chain test From b2c93f2da0880b0f5a0b671c03004c6ab343e420 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 16 Jan 2024 14:08:02 -0800 Subject: [PATCH 09/14] ci: simplify platform handling --- .github/workflows/ci.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 223fd87f..7bb5e844 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,25 +15,24 @@ concurrency: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - DEFAULT_PLATFORM: linux/amd64 + # Name these so they look less similar than AMD/ARM; omit 64 as uninformative. + X86_PLATFORM: linux/amd64 + ARM_PLATFORM: linux/arm64/v8 jobs: platforms: runs-on: ubuntu-latest outputs: - default: '${{ steps.platforms.outputs.default }}' platforms: '${{ steps.platforms.outputs.platforms }}' steps: - name: Compute Docker platforms id: platforms run: | - DEFAULT_PLATFORM=linux/amd64 - echo "default=$DEFAULT_PLATFORM" >> $GITHUB_OUTPUT if ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }}; then # JSON-encoded list consisting only of the default platform. - platforms='["'"$DEFAULT_PLATFORM"'"]' + platforms='["'"$X86_PLATFORM"'"]' else - platforms='["linux/amd64","linux/arm64/v8"]' + platforms='["$X86_PLATFORM","$ARM_PLATFORM"]' fi echo "platforms=$platforms" >> $GITHUB_OUTPUT @@ -63,10 +62,6 @@ jobs: echo "=== After cleanup:" df -h - - name: Set environment - run: | - echo "DEFAULT_PLATFORM=${{ needs.platforms.outputs.default }}" >> $GITHUB_ENV - - name: Checkout repository uses: actions/checkout@v4 @@ -119,7 +114,7 @@ jobs: docker info node_modules/.bin/synthetic-chain build - name: run proposal tests - if: ${{ matrix.platform == env.DEFAULT_PLATFORM }} + if: ${{ matrix.platform == env.X86_PLATFORM }} run: node_modules/.bin/synthetic-chain test # XXX this should be instant for the local platform because all the stages From c576608f6c01ebe2ceca968bc8ad36c39a51beeb Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 16 Jan 2024 14:38:05 -0800 Subject: [PATCH 10/14] docs: CI image building --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bb5e844..88323dcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: # were already built in the steps above but it's re-building the last # stage. This is deemed good enough for now. see # https://github.com/moby/moby/issues/34715 - - name: Build and push images + - name: Build and push default image uses: docker/build-push-action@v5 with: context: . @@ -132,7 +132,8 @@ jobs: tags: ${{ steps.docker-tags.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # Publish the build's multiarch images to Docker Registry. + # Merge the default image from each platform into one multi-arch image, + # then publish that multiarch image. docker-publish-multiarch: needs: [test-proposals, platforms] runs-on: ubuntu-latest @@ -156,7 +157,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Login to Docker Registry + - name: Log in to the Container registry uses: docker/login-action@v3 with: username: ${{ github.actor }} From 55cf5ad624f076f083e098e81929745ce1cb9783 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Wed, 17 Jan 2024 12:34:21 -0800 Subject: [PATCH 11/14] refactor: Bakefile matrix --- .gitignore | 1 + docker-bake.hcl | 35 +++++++++++++++++++ packages/synthetic-chain/cli.ts | 23 +++++++----- packages/synthetic-chain/src/cli/build.ts | 34 +++++++----------- .../synthetic-chain/src/cli/dockerfileGen.ts | 11 ++++++ 5 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 docker-bake.hcl diff --git a/.gitignore b/.gitignore index ad34e3d5..fc41e9eb 100644 --- a/.gitignore +++ b/.gitignore @@ -135,4 +135,5 @@ dist # build in CI Dockerfile +docker-bake.json /upgrade-test-scripts diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 00000000..98d336d0 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,35 @@ +// PROPOSALS variable is filled by docker-bake.json, which this config merges upon +// https://docs.docker.com/build/bake/reference/#file-format + +group "default" { + targets = [ + "use", + "test" + ] +} + +target "use" { + name = "use-${proposal}" + matrix = { + proposal = PROPOSALS + } + // TODO proposal *number* would be immutable + tags = ["ghcr.io/agoric/agoric-3-proposals:use-${proposal}"] + labels = { + "org.opencontainers.image.title": "Use ${proposal}", + "org.opencontainers.image.description": "Use agoric-3 synthetic chain after ${proposal} proposal", + } + target = "use-${proposal}" +} + +target "test" { + name = "test-${proposal}" + matrix = { + proposal = PROPOSALS + } + tags = ["ghcr.io/agoric/agoric-3-proposals:test-${proposal}"] + labels = { + "org.opencontainers.image.title": "Test ${proposal}", + } + target = "test-${proposal}" +} diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index 6940d676..8093f6c7 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -5,10 +5,13 @@ import path from 'node:path'; import { execSync } from 'node:child_process'; import { buildProposalSubmissions, - buildProposalImages, + bakeImages, readBuildConfig, } from './src/cli/build.js'; -import { writeDockerfile } from './src/cli/dockerfileGen.js'; +import { + writeBakefileProposals, + writeDockerfile, +} from './src/cli/dockerfileGen.js'; import { matchOneProposal, readProposals } from './src/cli/proposals.js'; import { debugTestImage, runTestImages } from './src/cli/run.js'; import { runDoctor } from './src/cli/doctor.js'; @@ -42,26 +45,30 @@ test [--debug] - build the "test" images and run them doctor - diagnostics and quick fixes `; -const buildUseImages = () => { +/** + * Put into places files that building depends upon. + */ +const prepareDockerBuild = () => { execSync( // XXX very brittle 'cp -r node_modules/@agoric/synthetic-chain/upgrade-test-scripts .', ); + writeDockerfile(allProposals, buildConfig.fromTag); + writeBakefileProposals(allProposals); buildProposalSubmissions(proposals); - buildProposalImages(proposals, 'use', values.dry); }; switch (cmd) { case 'build': { - const { fromTag } = buildConfig; - writeDockerfile(allProposals, fromTag); - buildUseImages(); + prepareDockerBuild(); + bakeImages('use', values.dry); break; } case 'test': // always rebuild all test images. Keeps it simple and these are fast // as long as the "use" stages are cached because they don't execute anything themselves. - buildProposalImages(proposals, 'test', values.dry); + prepareDockerBuild(); + bakeImages('test', values.dry); if (values.debug) { debugTestImage(matchOneProposal(proposals, match!)); } else { diff --git a/packages/synthetic-chain/src/cli/build.ts b/packages/synthetic-chain/src/cli/build.ts index b7459969..8843dda3 100755 --- a/packages/synthetic-chain/src/cli/build.ts +++ b/packages/synthetic-chain/src/cli/build.ts @@ -1,7 +1,7 @@ import { execSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; -import { ProposalInfo, imageNameForProposal } from './proposals.js'; +import { ProposalInfo } from './proposals.js'; export type AgoricSyntheticChainConfig = { /** @@ -62,25 +62,15 @@ export const buildProposalSubmissions = (proposals: ProposalInfo[]) => { } }; -export const buildProposalImages = ( - proposals: ProposalInfo[], - stage: 'test' | 'use', - dry = false, -) => { - for (const proposal of proposals) { - if (!dry) { - console.log( - `\nBuilding test image for proposal ${proposal.proposalName}`, - ); - } - const { name, target } = imageNameForProposal(proposal, stage); - // 'load' to ensure the images are output to the Docker client. Seems to be necessary - // for the CI docker/build-push-action to re-use the cached stages. - const cmd = `docker buildx build --load --tag ${name} --target ${target} .`; - console.log(cmd); - if (!dry) { - // `time` to output how long each build takes - execSync(`time ${cmd}`, { stdio: 'inherit' }); - } - } +/** + * Bake images using the docker buildx bake command. + * + * @param matrix - The group target + * @param [dry] - Whether to skip building and just print the build config. + */ +export const bakeImages = (matrix: 'test' | 'use', dry = false) => { + // https://docs.docker.com/engine/reference/commandline/buildx_build/#load + const cmd = `docker buildx bake --load ${matrix} ${dry ? '--print' : ''}`; + console.log(cmd); + execSync(cmd, { stdio: 'inherit' }); }; diff --git a/packages/synthetic-chain/src/cli/dockerfileGen.ts b/packages/synthetic-chain/src/cli/dockerfileGen.ts index 2ccad1ee..35015c22 100755 --- a/packages/synthetic-chain/src/cli/dockerfileGen.ts +++ b/packages/synthetic-chain/src/cli/dockerfileGen.ts @@ -181,6 +181,17 @@ ENTRYPOINT ./start_agd.sh }, }; +export function writeBakefileProposals(allProposals: ProposalInfo[]) { + const json = { + variable: { + PROPOSALS: { + default: allProposals.map(p => p.proposalName), + }, + }, + }; + fs.writeFileSync('docker-bake.json', JSON.stringify(json, null, 2)); +} + export function writeDockerfile( allProposals: ProposalInfo[], fromTag?: string | null, From 43ef6141b683aa0f74bb5ab156f4a872015b25e2 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Fri, 19 Jan 2024 11:18:39 -0800 Subject: [PATCH 12/14] docs: CLI usage --- README.md | 2 ++ packages/synthetic-chain/README.md | 11 +++++++---- packages/synthetic-chain/cli.ts | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1ce5f804..16f08947 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ If the proposal is _pending_ and does not yet have a number, use a letter. The p ## Development +A known issue is that `yarn synthetic-chain` files with `Unknown file extension ".ts"`. To work around it, run from the bin dir as below. + To build the test images, ``` diff --git a/packages/synthetic-chain/README.md b/packages/synthetic-chain/README.md index 489d5a24..f9880974 100644 --- a/packages/synthetic-chain/README.md +++ b/packages/synthetic-chain/README.md @@ -2,12 +2,15 @@ Utilities to build a synthetic chain and test running proposals atop it. The chain approximates agoric-3 (Mainnet) using the state from https://github.com/Agoric/agoric-3-proposals (It could trivially support other Agoric chains, if we scale horizontally.) -```sh -node_modules/.bin/synthetic-chain build +## Usage -node_modules/.bin/synthetic-chain test +``` +build - build the synthetic-chain "use" images + +test [--debug] - build the "test" images and run them +test -m - target a particular proposal by substring match -node_modules/.bin/synthetic-chain test --debug -m +doctor - diagnostics and quick fixes ``` ## Design diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index 8093f6c7..5e3974bc 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -41,6 +41,7 @@ const usage = `USAGE: build - build the synthetic-chain "use" images test [--debug] - build the "test" images and run them +test -m - target a particular proposal by substring match doctor - diagnostics and quick fixes `; From 5a87dfef5dc9dec4dd795e79ebcc6b199f1067d4 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Wed, 17 Jan 2024 12:41:47 -0800 Subject: [PATCH 13/14] ci: push proposal 'use' images --- .github/workflows/ci.yml | 25 ++++++++++++++++++++----- docker-bake.hcl | 7 +++++++ packages/synthetic-chain/cli.ts | 4 ++-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88323dcc..a0862a30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,15 +108,30 @@ jobs: # Enable corepack for packageManager config - run: corepack enable || sudo corepack enable - run: yarn install + # Set up docker-bake files used by docker/bake-action + - run: node_modules/.bin/synthetic-chain prepare-ci - - name: build proposal images - run: | - docker info - node_modules/.bin/synthetic-chain build - - name: run proposal tests + # Test before pushing the images. + - name: Build and run proposal tests if: ${{ matrix.platform == env.X86_PLATFORM }} run: node_modules/.bin/synthetic-chain test + # Build a "use" image for each proposal. This uses Docker Bake's + # matrix feature. We could have each "use" image built in a different runner + # by including https://github.com/docker/bake-action?tab=readme-ov-file#list-targets + # in the GHA matrix, but that wouldn't be able to resolve the DAG of what to build first. + - name: Push proposal "use" images + uses: docker/bake-action@v4 + # If we pushed from PRs, each one would overwrite main's (e.g. use-upgrade-8) + # To push PR "use" images we'll need to qualify the tag (e.g. use-upgrade-8-pr-2). + if: ${{ github.event_name != 'pull_request' }} + with: + files: | + ./docker-bake.json + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: use + # XXX this should be instant for the local platform because all the stages # were already built in the steps above but it's re-building the last # stage. This is deemed good enough for now. see diff --git a/docker-bake.hcl b/docker-bake.hcl index 98d336d0..75f2e728 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,3 +1,9 @@ +// If you're new to Bake: +// https://blog.kubesimplify.com/bake-your-container-images-with-bake + +// for https://github.com/docker/metadata-action?tab=readme-ov-file#bake-definition +target "docker-metadata-action" {} + // PROPOSALS variable is filled by docker-bake.json, which this config merges upon // https://docs.docker.com/build/bake/reference/#file-format @@ -9,6 +15,7 @@ group "default" { } target "use" { + inherits = ["docker-metadata-action"] name = "use-${proposal}" matrix = { proposal = PROPOSALS diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index 5e3974bc..987c9a01 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -66,8 +66,8 @@ switch (cmd) { break; } case 'test': - // always rebuild all test images. Keeps it simple and these are fast - // as long as the "use" stages are cached because they don't execute anything themselves. + // Always rebuild all test images to keep it simple. With the "use" stages + // cached, these are pretty fast building doesn't run agd. prepareDockerBuild(); bakeImages('test', values.dry); if (values.debug) { From 9285e9bbf288df003812c7c9e89bef87c812e016 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 18 Jan 2024 10:43:06 -0800 Subject: [PATCH 14/14] ci: re-use penultimate stage build --- .github/workflows/ci.yml | 4 ---- packages/synthetic-chain/src/cli/dockerfileGen.ts | 8 +++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0862a30..98056cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,10 +132,6 @@ jobs: ${{ steps.meta.outputs.bake-file }} targets: use - # XXX this should be instant for the local platform because all the stages - # were already built in the steps above but it's re-building the last - # stage. This is deemed good enough for now. see - # https://github.com/moby/moby/issues/34715 - name: Build and push default image uses: docker/build-push-action@v5 with: diff --git a/packages/synthetic-chain/src/cli/dockerfileGen.ts b/packages/synthetic-chain/src/cli/dockerfileGen.ts index 35015c22..00a45d7f 100755 --- a/packages/synthetic-chain/src/cli/dockerfileGen.ts +++ b/packages/synthetic-chain/src/cli/dockerfileGen.ts @@ -8,6 +8,7 @@ import { type ProposalInfo, type SoftwareUpgradeProposal, encodeUpgradeInfo, + imageNameForProposal, } from './proposals.js'; /** @@ -170,9 +171,14 @@ ENTRYPOINT ./run_test.sh ${proposalIdentifier}:${proposalName} * The last target in the file, for untargeted `docker build` */ DEFAULT(lastProposal: ProposalInfo) { + // Assumes the 'use' image is built and tagged. + // This isn't necessary for a multi-stage build, but without it CI + // rebuilds the last "use" image during the "default" image step + // Some background: https://github.com/moby/moby/issues/34715 + const useImage = imageNameForProposal(lastProposal, 'use').name; return ` # DEFAULT -FROM use-${lastProposal.proposalName} +FROM ${useImage} WORKDIR /usr/src/upgrade-test-scripts SHELL ["/bin/bash", "-c"]