|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Builds linux deb and rpm repositories and uploads them to a repository server. |
| 4 | +# One instance of this script targets *one* server environment. |
| 5 | +# This means that if you want to publish to development, staging and production servers, |
| 6 | +# you need to run three instances of this script. |
| 7 | +# |
| 8 | +# This script works on an $inbox_dir. In this directory it expects one directory per repository. |
| 9 | +# For each repository it will read all files having the .src extension |
| 10 | +# These files are expected to contain a single line, a path to some directory where |
| 11 | +# it can read new artifacts for that product. |
| 12 | +# All deb and rpm files from that directory will be signed and moved over to a folder with |
| 13 | +# the same name as the .src file, but with a .latest extension instead. |
| 14 | +# So artifacts read from `app.src` will be moved to `app.latest/`. |
| 15 | +# |
| 16 | +# Then the deb and rpm repositories will be generated and all deb and rpm files in |
| 17 | +# $inbox_dir/$repository/*.latest/ will be added to the repository. These repositories are then synced |
| 18 | +# to `$repository_server_upload_domain respectively. |
| 19 | + |
| 20 | +set -eu |
| 21 | +# nullglob is needed to produce expected results when globing an empty directory |
| 22 | +shopt -s nullglob |
| 23 | + |
| 24 | +function usage() { |
| 25 | + echo "Usage: $0 <environment>" |
| 26 | + echo |
| 27 | + echo "Example usage: ./build-linux-repositories.sh production" |
| 28 | + echo |
| 29 | + echo "This script reads an inbox folder for the corresponding server environment." |
| 30 | + echo "It then generates and uploads new Linux repositories for all" |
| 31 | + echo "the latest artifacts" |
| 32 | + echo |
| 33 | + echo "Options:" |
| 34 | + echo " -h | --help Show this help message and exit." |
| 35 | + exit 1 |
| 36 | +} |
| 37 | + |
| 38 | +if [[ "$#" == 0 || $1 == "-h" || $1 == "--help" ]]; then |
| 39 | + usage |
| 40 | +fi |
| 41 | + |
| 42 | +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" |
| 43 | + |
| 44 | +# shellcheck source=ci/linux-repository-builder/build-linux-repositories-config.sh |
| 45 | +source "$SCRIPT_DIR/build-linux-repositories-config.sh" |
| 46 | + |
| 47 | +environment="$1" |
| 48 | +case "$environment" in |
| 49 | + "production") |
| 50 | + repository_server_upload_domain="$PRODUCTION_REPOSITORY_SERVER" |
| 51 | + repository_server_public_url="$PRODUCTION_LINUX_REPOSITORY_PUBLIC_URL" |
| 52 | + ;; |
| 53 | + "staging") |
| 54 | + repository_server_upload_domain="$STAGING_REPOSITORY_SERVER" |
| 55 | + repository_server_public_url="$STAGING_LINUX_REPOSITORY_PUBLIC_URL" |
| 56 | + ;; |
| 57 | + "dev") |
| 58 | + repository_server_upload_domain="$DEV_REPOSITORY_SERVER" |
| 59 | + repository_server_public_url="$DEV_LINUX_REPOSITORY_PUBLIC_URL" |
| 60 | + ;; |
| 61 | + *) |
| 62 | + echo "Unknown environment. Specify production, staging or dev" >&2 |
| 63 | + exit 1 |
| 64 | + ;; |
| 65 | +esac |
| 66 | + |
| 67 | +inbox_dir="$LINUX_REPOSITORY_INBOX_DIR_BASE/$environment" |
| 68 | + |
| 69 | +if [[ ! -d "$inbox_dir" ]]; then |
| 70 | + echo "Inbox $inbox_dir does not exist" 1>&2 |
| 71 | + exit 1 |
| 72 | +fi |
| 73 | + |
| 74 | +# Process all .src files in the given inbox dir. |
| 75 | +# Signs and moves all found artifacts into .latest directories |
| 76 | +# Returns 0 if everything went well and there are new artifacts for a product. |
| 77 | +# Returns 1 if no new artifacts were found |
| 78 | +function process_inbox { |
| 79 | + local inbox_dir=$1 |
| 80 | + echo "[#] Processing inbox at $inbox_dir" |
| 81 | + |
| 82 | + local found_new_artifacts="false" |
| 83 | + # Read all notify files and move the artifacts they point to into a local .latest copy |
| 84 | + for notify_file in "$inbox_dir"/*.src; do |
| 85 | + if [[ ! -f "$notify_file" ]]; then |
| 86 | + echo "Ignoring non-file $notify_file" 1>&2 |
| 87 | + continue |
| 88 | + fi |
| 89 | + echo "Processing notify file $notify_file" |
| 90 | + |
| 91 | + local src_dir |
| 92 | + src_dir=$(cat "$notify_file") |
| 93 | + if [[ ! -d "$src_dir" ]]; then |
| 94 | + echo "Artifact source dir $src_dir from notify file $notify_file does not exist" 1>&2 |
| 95 | + continue |
| 96 | + fi |
| 97 | + |
| 98 | + # Removing the file before we move the artifacts out of where it points. |
| 99 | + # This ensures we don't get stuck in a loop processing it over and over |
| 100 | + # if the signing and moving fails. |
| 101 | + rm "$notify_file" |
| 102 | + |
| 103 | + local artifact_dir=${notify_file/%.src/.latest} |
| 104 | + # Recreate the artifact dir, cleaning it |
| 105 | + rm -rf "$artifact_dir" && mkdir -p "$artifact_dir" || exit 1 |
| 106 | + |
| 107 | + # The fact that we have processed one .src file is enough tro trigger a repository |
| 108 | + # rebuild. Because if a product would like to publish "no artifacts" they should |
| 109 | + # be able to create a .src file pointing to an empty directory |
| 110 | + found_new_artifacts="true" |
| 111 | + |
| 112 | + echo "Moving artifacts from $src_dir to $artifact_dir" |
| 113 | + # Move all deb and rpm files into the .latest dir |
| 114 | + for src_deb in "$src_dir"/*.deb; do |
| 115 | + echo "Signing and moving $src_deb into $artifact_dir/" |
| 116 | + dpkg-sig --sign builder "$src_deb" |
| 117 | + mv "$src_deb" "$artifact_dir/" |
| 118 | + done |
| 119 | + for src_rpm in "$src_dir"/*.rpm; do |
| 120 | + echo "Signing and moving $src_rpm into $artifact_dir/" |
| 121 | + rpm --addsign "$src_rpm" |
| 122 | + mv "$src_rpm" "$artifact_dir/" |
| 123 | + done |
| 124 | + rm -r "$src_dir" || echo "Failed to remove src dir $src_dir" |
| 125 | + done |
| 126 | + |
| 127 | + if [[ $found_new_artifacts == "false" ]]; then |
| 128 | + return 1 |
| 129 | + fi |
| 130 | + return 0 |
| 131 | +} |
| 132 | + |
| 133 | +function rsync_repo { |
| 134 | + local local_repo_dir=$1 |
| 135 | + local remote_repo_dir=$2 |
| 136 | + |
| 137 | + echo "Syncing to $repository_server_upload_domain:$remote_repo_dir" |
| 138 | + rsync -av --delete --mkpath --rsh='ssh -p 1122' \ |
| 139 | + "$local_repo_dir"/ \ |
| 140 | + build@"$repository_server_upload_domain":"$remote_repo_dir" |
| 141 | +} |
| 142 | + |
| 143 | +for repository in "${REPOSITORIES[@]}"; do |
| 144 | + deb_remote_repo_dir="deb/$repository" |
| 145 | + rpm_remote_repo_dir="rpm/$repository" |
| 146 | + |
| 147 | + repository_inbox_dir="$inbox_dir/$repository" |
| 148 | + if ! process_inbox "$repository_inbox_dir"; then |
| 149 | + echo "Nothing new in inbox at $repository_inbox_dir" |
| 150 | + continue |
| 151 | + fi |
| 152 | + |
| 153 | + # Read all .latest artifact dirs into array |
| 154 | + readarray -d '' artifact_dirs < <(find "$repository_inbox_dir" -maxdepth 1 -name "*.latest" -type d -print0) |
| 155 | + if [ "${#artifact_dirs[@]}" -lt 1 ]; then |
| 156 | + echo "No artifact directories in $repository_inbox_dir to generate repositories from" >&2 |
| 157 | + continue |
| 158 | + fi |
| 159 | + |
| 160 | + echo "Generating repositories from these artifact directories: ${artifact_dirs[*]}" |
| 161 | + |
| 162 | + # Generate deb repository from all the .latest artifacts |
| 163 | + |
| 164 | + deb_repo_dir="$repository_inbox_dir/repos/deb" |
| 165 | + rm -rf "$deb_repo_dir" && mkdir -p "$deb_repo_dir" || exit 1 |
| 166 | + "$SCRIPT_DIR/prepare-apt-repository.sh" "$deb_repo_dir" "${artifact_dirs[@]}" |
| 167 | + |
| 168 | + # Generate rpm repository from all the .latest artifacts |
| 169 | + |
| 170 | + rpm_repo_dir="$repository_inbox_dir/repos/rpm" |
| 171 | + rm -rf "$rpm_repo_dir" && mkdir -p "$rpm_repo_dir" || exit 1 |
| 172 | + "$SCRIPT_DIR/prepare-rpm-repository.sh" "$rpm_repo_dir" \ |
| 173 | + "$repository_server_public_url" "$rpm_remote_repo_dir" "$repository" "${artifact_dirs[@]}" |
| 174 | + |
| 175 | + # rsync repositories to repository server |
| 176 | + |
| 177 | + echo "[#] Syncing deb repository to $deb_remote_repo_dir" |
| 178 | + rsync_repo "$deb_repo_dir" "$deb_remote_repo_dir" |
| 179 | + echo "[#] Syncing rpm repository to $rpm_remote_repo_dir" |
| 180 | + rsync_repo "$rpm_repo_dir" "$rpm_remote_repo_dir" |
| 181 | + |
| 182 | +done |
0 commit comments