Skip to content

Commit a563f08

Browse files
licy183licy183
committed
ci: reuse pr artifacts
1 parent c32e52b commit a563f08

File tree

2 files changed

+113
-23
lines changed

2 files changed

+113
-23
lines changed

‎.github/workflows/packages.yml

Lines changed: 48 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -59,30 +59,50 @@ jobs:
5959
run: ./setup-environment.sh
6060

6161
- name: Gather build summary
62+
id: build-info
63+
env:
64+
MANUAL_INPUT_PACKAGES: ${{ github.event.inputs.packages }}
6265
run: |
66+
# We are intentionally not using .commits[0].id and .commits[-1].id as github seems to
67+
# only send 20 commits in the payload for github action runs instead of the 2048 documented
68+
# limit. Perhaps 2048 is the limit just for webhooks, where they haven't documented
69+
# properly that the limit for github actions is only 20:
70+
#
71+
# https://docs.github.com/en/webhooks/webhook-events-and-payloads#push
72+
OLD_COMMIT="${{ github.event.before }}"
73+
HEAD_COMMIT="${{ github.event.after }}"
6374
if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
64-
BASE_COMMIT=$(jq --raw-output .pull_request.base.sha "$GITHUB_EVENT_PATH")
65-
OLD_COMMIT=$(jq --raw-output .commits[0].id "$GITHUB_EVENT_PATH")
66-
HEAD_COMMIT=$(jq --raw-output .commits[-1].id "$GITHUB_EVENT_PATH")
67-
if [ "$BASE_COMMIT" = "null" ]; then
68-
if [ "$OLD_COMMIT" = "$HEAD_COMMIT" ]; then
69-
# Single-commit push.
70-
echo "Processing commit: ${HEAD_COMMIT}"
71-
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${HEAD_COMMIT}")
72-
else
73-
# Multi-commit push.
74-
OLD_COMMIT="${OLD_COMMIT}~1"
75-
echo "Processing commit range: ${OLD_COMMIT}..${HEAD_COMMIT}"
76-
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${OLD_COMMIT}" "${HEAD_COMMIT}")
75+
BASE_REF="${{ github.event.pull_request.base.ref }}"
76+
git fetch origin "${BASE_REF:-master}" 2>/dev/null
77+
BASE_COMMIT="$(git merge-base "origin/${BASE_REF:-master}" "HEAD")"
78+
if [ -z "${{ github.event.pull_request.base.sha }}" ]; then
79+
if ! git log "$OLD_COMMIT" > /dev/null; then
80+
if [ "$(git branch --show-current)" = "master" ]; then
81+
echo "Force push detected on master branch. Unable to proceed."
82+
exit 1
83+
else
84+
OLD_COMMIT=$(git fetch origin master >&2; git merge-base origin/master $HEAD_COMMIT)
85+
fi
7786
fi
87+
echo "Processing commit range: ${OLD_COMMIT}..${HEAD_COMMIT}"
88+
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${OLD_COMMIT}" "${HEAD_COMMIT}")
7889
else
7990
# Pull requests.
80-
echo "Processing pull request #$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"): ${BASE_COMMIT}..HEAD"
91+
echo "Processing pull request #${{ github.event.pull_request.number }}: ${BASE_COMMIT}..HEAD"
8192
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${BASE_COMMIT}" "HEAD")
8293
fi
8394
fi
8495
mkdir -p ./artifacts ./debs
8596
touch ./debs/.placeholder
97+
98+
if [[ "${{ github.event_name }}" == "pull_request" && -n "$(git rev-list --merges "$(git fetch origin master >&2; git merge-base origin/master $HEAD_COMMIT)..$HEAD_COMMIT")" ]]; then
99+
# Github does not allow multiline errors, but it will interpret the escape sequence %0A as a line break.
100+
echo "::error ::Merge commits are not allowed in pull requests.%0AYou should rebase your commits or squash them.%0Ahttps://docs.github.com/en/get-started/using-git/using-git-rebase-on-the-command-line"
101+
exit 1
102+
fi
103+
104+
GITHUB_OUTPUT="$GITHUB_OUTPUT" OLD_COMMIT="$OLD_COMMIT" HEAD_COMMIT="$HEAD_COMMIT" GITHUB_TOKEN="${{ secrets.GH_TOKEN }}" scripts/utils/termux_reuse_pr_build_artifacts.sh "${{ github.event_name }}" "${{ matrix.target_arch }}" || true
105+
86106
if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
87107
# Process tag '%ci:no-build' that may be added as line to commit message.
88108
# Forces CI to cancel current build with status 'passed'
@@ -134,7 +154,10 @@ jobs:
134154
done<<<${CHANGED_FILES}
135155
done
136156
else
137-
for pkg in ${{ github.event.inputs.packages }}; do
157+
# Ensure MANUAL_INPUT_PACKAGES is newline free, and put it
158+
# into an array
159+
read -a PACKAGES <<< "${MANUAL_INPUT_PACKAGES//$'\n'/ }"
160+
for pkg in "${PACKAGES[@]}"; do
138161
repo_paths=$(jq --raw-output 'keys | .[]' repo.json)
139162
found=false
140163
for repo_path in $repo_paths; do
@@ -168,14 +191,6 @@ jobs:
168191
mv ./deleted_${repo}_packages.txt.tmp ./deleted_${repo}_packages.txt
169192
fi
170193
done
171-
- name: Free additional disk space (if necessary)
172-
run: |
173-
if grep -Eq "^($(paste -s -d '|' ./big-pkgs.list))$" ./built_tur_packages.txt; then
174-
sudo apt purge -yq $(dpkg -l | grep '^ii' | awk '{ print $2 }' | grep -P '(aspnetcore|cabal-|dotnet-|ghc-|libmono|mongodb-|mysql-|php)') \
175-
firefox google-chrome-stable microsoft-edge-stable mono-devel mono-runtime-common monodoc-manual ruby
176-
sudo apt autoremove -yq
177-
sudo rm -rf /opt/hostedtoolcache /usr/local /usr/share/dotnet /usr/share/swift
178-
fi
179194
- name: Lint packages
180195
run: |
181196
declare -a package_recipes
@@ -188,7 +203,17 @@ jobs:
188203
if [ ! -z "$package_recipes" ]; then
189204
./scripts/lint-packages.sh $package_recipes
190205
fi
206+
- name: Free additional disk space (if necessary)
207+
if: ${{ steps.build-info.outputs.skip-building != 'true' }}
208+
run: |
209+
if grep -Eq "^($(paste -s -d '|' ./big-pkgs.list))$" ./built_tur_packages.txt; then
210+
sudo apt purge -yq $(dpkg -l | grep '^ii' | awk '{ print $2 }' | grep -P '(aspnetcore|cabal-|dotnet-|ghc-|libmono|mongodb-|mysql-|php)') \
211+
firefox google-chrome-stable microsoft-edge-stable mono-devel mono-runtime-common monodoc-manual ruby
212+
sudo apt autoremove -yq
213+
sudo rm -rf /opt/hostedtoolcache /usr/local /usr/share/dotnet /usr/share/swift
214+
fi
191215
- name: Build packages
216+
if: ${{ steps.build-info.outputs.skip-building != 'true' }}
192217
run: |
193218
declare -a packages
194219
for repo_path in $(jq --raw-output 'keys | .[]' repo.json); do

‎common-files/building-system-patches/0008-allow-reuse-pr.patch

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
--- a/scripts/utils/termux_reuse_pr_build_artifacts.sh
2+
+++ b/scripts/utils/termux_reuse_pr_build_artifacts.sh
3+
@@ -38,7 +38,7 @@
4+
curl --silent \
5+
-H "Authorization: token ${GITHUB_TOKEN}" \
6+
-H "Accept: application/vnd.github.v3+json" \
7+
- "https://api.github.com/repos/termux/termux-packages/actions/runs/${1}/artifacts" \
8+
+ "https://api.github.com/repos/termux-user-repository/tur/actions/runs/${1}/artifacts" \
9+
| jq -r '[.artifacts[]? | select(.name | startswith("debs-'"${TARGET_ARCH}"'")) | .archive_download_url][0] // error' \
10+
|| return $?
11+
}
12+
@@ -92,7 +92,7 @@
13+
# We should obtain data about all commits in this push to check that they are from the same PR if any
14+
RELATED_PRS_QUERY="
15+
query {
16+
- repository(owner: \"termux\", name: \"termux-packages\") {
17+
+ repository(owner: \"termux-user-repository\", name: \"tur\") {
18+
$(
19+
for commit in "${COMMITS[@]}"; do
20+
# Add a query for this commit with the shorthash as the label
21+
@@ -124,7 +124,7 @@
22+
.edges[0].node.body)' <<< "$RESPONSE" || :)
23+
[[ -n "${PR_HEAD_COMMIT:-}" ]] || infoexit "failed to read associated PR head commit, not performing CI fast path"
24+
25+
- echo "::group::Detected PR #${PRS[0]}: ${PR_COMMIT_TITLE} — https://github.com/termux/termux-packages/pull/${PRS[0]}"
26+
+ echo "::group::Detected PR #${PRS[0]}: ${PR_COMMIT_TITLE} — https://github.com/termux-user-repository/tur/pull/${PRS[0]}"
27+
echo "${PR_COMMIT_BODY}"
28+
echo "::endgroup::"
29+
30+
@@ -206,7 +206,7 @@
31+
# The same commit can be used in more than one PR or even push
32+
WORKFLOW_PR_QUERY="
33+
query {
34+
- repository(owner: \"termux\", name: \"termux-packages\") {
35+
+ repository(owner: \"termux-user-repository\", name: \"tur\") {
36+
object(oid: \"$PR_HEAD_COMMIT\") { ...workflowRun }
37+
}
38+
}
39+
@@ -228,7 +228,7 @@
40+
| .workflowRun.databaseId][0] // empty' <<< "$RESPONSE" || :
41+
)"
42+
if [[ -n "${WORKFLOW_ID}" ]]; then
43+
- echo "We can safely reuse CI artifacts from https://github.com/termux/termux-packages/actions/runs/${WORKFLOW_ID}"
44+
+ echo "We can safely reuse CI artifacts from https://github.com/termux-user-repository/tur/actions/runs/${WORKFLOW_ID}"
45+
if download_ci_artifacts "${WORKFLOW_ID}"; then
46+
# Notify CI about skipping packages building because we reuse PR artifact.
47+
echo "skip-building=true" >> "${GITHUB_OUTPUT:-/dev/null}"
48+
@@ -253,7 +253,7 @@
49+
# to allow CI reuse artifacts of other PRs in the case if current PR diverged from another PR branch.
50+
WORKFLOW_COMMITS_QUERY="
51+
query {
52+
- repository(owner: \"termux\", name: \"termux-packages\") {
53+
+ repository(owner: \"termux-user-repository\", name: \"tur\") {
54+
$(
55+
for commit in "${COMMITS[@]}"; do
56+
# Add a query for this commit with the shorthash as the label
57+
@@ -285,7 +285,7 @@
58+
[[ -z "${WORKFLOW_ID:-}" ]] || break
59+
done
60+
if [[ -n "${WORKFLOW_ID}" ]]; then
61+
- echo "We can safely reuse CI artifacts from https://github.com/termux/termux-packages/actions/runs/${WORKFLOW_ID}"
62+
+ echo "We can safely reuse CI artifacts from https://github.com/termux-user-repository/tur/actions/runs/${WORKFLOW_ID}"
63+
echo "CI artifact URL is $(ci_artifact_url "${WORKFLOW_ID}" || infoexit "Failed to get CI artifact URL")"
64+
else
65+
echo "We can not reuse CI artifacts since no relevant CI runs were found"

0 commit comments

Comments
 (0)