From 995394cc24020c2ffedf5fb53aa82f5bcd84f7c0 Mon Sep 17 00:00:00 2001 From: James Anslow Date: Tue, 28 Jan 2020 16:39:39 +0000 Subject: [PATCH 01/23] Fixing a bug whereby build list showed erroneous console output regarding regions --- cli/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/build.go b/cli/build.go index 0ea62d29..cbda7a04 100644 --- a/cli/build.go +++ b/cli/build.go @@ -166,7 +166,7 @@ var buildListCmd = &cobra.Command{ log.Fatalf("Failed to setup AWS session: %v", err) } - log.Infof("Looking for builds for stack %v in the following regions: %v", name, instanceRegions) + log.Infof("Looking for builds for stack %v in the following regions: %v", name, listRegions) runningInstances := 0 for _, region := range strings.Split(listRegions, ",") { ec2Client := ec2.New(sess, &aws.Config{Region: ®ion}) From c71b6300686cff92736a85c4ca3a5930c62ba4d0 Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 2 Feb 2020 19:00:09 -0800 Subject: [PATCH 02/23] fix issue #145 - where network stack keys were not being copied --- templates/build_template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/build_template.go b/templates/build_template.go index 56088a35..50eb1580 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -1199,7 +1199,7 @@ aws_import_keys() { aws s3 sync "${KEYS_DIR}" "s3://${AWS_ENCRYPTED_KEYS_BUCKET}" --exclude "*" --include "*.gpg" else log "Uploading new networkstack key to s3://${AWS_KEYS_BUCKET}" - aws s3 sync "s3://${AWS_KEYS_BUCKET}" "${KEYS_DIR}" + aws s3 sync "${KEYS_DIR}" "s3://${AWS_KEYS_BUCKET}" fi fi popd From 79055bb486a44473becfb9110ba9e7ae5dc974c5 Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 2 Feb 2020 19:03:38 -0800 Subject: [PATCH 03/23] fix issue #148 - was not properly getting latest fdroid versions. also included alpha client releases in hopes of getting fix for privileged extension not working --- templates/build_template.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index 50eb1580..af619899 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -157,12 +157,13 @@ get_latest_versions() { fi # fdroid - get latest non alpha tags from gitlab (sorted) - FDROID_CLIENT_VERSION=$(curl --fail -s "$FDROID_CLIENT_URL_LATEST" | jq -r 'sort_by(.name) | reverse | [.[] | select(.name | test("^[0-9]+\\.[0-9]+")) | select(.name | contains("alpha") | not) | select(.name | contains("ota") | not)][0] | .name') + # TODO: exclude alpha once 1.8 stable is released + FDROID_CLIENT_VERSION=$(curl --fail -s "$FDROID_CLIENT_URL_LATEST" | jq -r '[.[] | select(.name | test("^[0-9]+\\.[0-9]+")) | select(.name | contains("ota") | not)][] | .name' | sort --version-sort -r | head -1) if [ -z "$FDROID_CLIENT_VERSION" ]; then aws_notify_simple "ERROR: Unable to get latest F-Droid version details. Stopping build." exit 1 fi - FDROID_PRIV_EXT_VERSION=$(curl --fail -s "$FDROID_PRIV_EXT_URL_LATEST" | jq -r 'sort_by(.name) | reverse | [.[] | select(.name | test("^[0-9]+\\.[0-9]+")) | select(.name | contains("alpha") | not) | select(.name | contains("ota") | not)][0] | .name') + FDROID_PRIV_EXT_VERSION=$(curl --fail -s "$FDROID_PRIV_EXT_URL_LATEST" | jq -r '[.[] | select(.name | test("^[0-9]+\\.[0-9]+")) | select(.name | contains("alpha") | not) | select(.name | contains("ota") | not)][] | .name' | sort --version-sort -r | head -1) if [ -z "$FDROID_PRIV_EXT_VERSION" ]; then aws_notify_simple "ERROR: Unable to get latest F-Droid privilege extension version details. Stopping build." exit 1 From 606e825f15950d5fa12a1af9a035f9f078c453c5 Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 2 Feb 2020 19:05:04 -0800 Subject: [PATCH 04/23] enable doze mode - closes #120 --- templates/build_template.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/build_template.go b/templates/build_template.go index af619899..d278f772 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -846,6 +846,9 @@ patch_base_config() { # enable swipe up gesture functionality as option sed -i 's@false@true@' ${BUILD_DIR}/frameworks/base/core/res/res/values/config.xml + + # enable doze and app standby + sed -i 's@false@true@' ${BUILD_DIR}/frameworks/base/core/res/res/values/config.xml } patch_settings_app() { From f080fb8f2b7df37a75a86b2996873a4a2dc67cd8 Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 2 Feb 2020 19:06:14 -0800 Subject: [PATCH 05/23] swap ordering of chromium build to browser first then webview as temporary workaround for #151 --- templates/build_template.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index d278f772..038006ad 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -628,10 +628,11 @@ android_default_version_code = "$DEFAULT_VERSION" EOF gn gen out/Default - log "Building chromium system_webview_apk target" - autoninja -C out/Default/ system_webview_apk log "Building chromium chrome_modern_public_apk target" autoninja -C out/Default/ chrome_modern_public_apk + + log "Building chromium system_webview_apk target" + autoninja -C out/Default/ system_webview_apk # upload to s3 for future builds aws s3 cp "out/Default/apks/SystemWebView.apk" "s3://${AWS_RELEASE_BUCKET}/chromium/SystemWebView.apk" From 788f45ecceb4bed2186efef71f840cb7dd48885a Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 2 Feb 2020 19:14:35 -0800 Subject: [PATCH 06/23] move platform_external_chromium back to 10.0 branch --- templates/build_template.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index 038006ad..25ef9400 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -651,7 +651,6 @@ aosp_repo_modifications() { log_header ${FUNCNAME} cd "${BUILD_DIR}" - # TODO: remove revision=dev from platform_external_chromium in future release, didn't want to break build for anyone on beta 10.x build # make modifications to default AOSP if ! grep -q "RattlesnakeOS" .repo/manifest.xml; then # really ugly awk script to add additional repos to manifest @@ -676,7 +675,7 @@ aosp_repo_modifications() { <% if .EnableAttestation %> print " "; <% end %> - print " "; + print " "; print " "; print " "; print " "; From 7d941688e04f5ca00c7da88e24f7b6a2edce12eb Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 2 Feb 2020 19:19:48 -0800 Subject: [PATCH 07/23] bump version to 10.0.2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 15324205..7e2df97f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.1 +10.0.2 From 75737f653aeafddcbc6a7c2da35d8cc4528a439b Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 16 Feb 2020 19:12:02 -0800 Subject: [PATCH 08/23] chromium dev channel is at v82 now and has build errors. now that stable chromium is v80, revert channel from dev back to stable --- VERSION | 2 +- templates/build_template.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 7e2df97f..6a7144d3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.2 +10.0.3 diff --git a/templates/build_template.go b/templates/build_template.go index 25ef9400..78b8d071 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -107,7 +107,7 @@ SECONDS=0 BUILD_TARGET="release aosp_${DEVICE} ${BUILD_TYPE}" RELEASE_URL="https://${AWS_RELEASE_BUCKET}.s3.amazonaws.com" RELEASE_CHANNEL="${DEVICE}-${BUILD_CHANNEL}" -CHROME_CHANNEL="dev" +CHROME_CHANNEL="stable" BUILD_DATE=$(date +%Y.%m.%d.%H) BUILD_TIMESTAMP=$(date +%s) BUILD_DIR="$HOME/rattlesnake-os" From 0b200ca888f3be1b4f594d3a7bcd23cc540aecce Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Fri, 28 Feb 2020 01:25:54 -0800 Subject: [PATCH 09/23] fix build break due to upstream repo change: https://gerrit.googlesource.com/git-repo/+/a269b1cb9dc21dfd598bfea3766206b606ad4589. switch over to creating a local manifest rather than direct modification of manifest with awk/sed. --- VERSION | 2 +- templates/build_template.go | 71 ++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/VERSION b/VERSION index 6a7144d3..6b48f258 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.3 +10.0.4 diff --git a/templates/build_template.go b/templates/build_template.go index 78b8d071..351fa32f 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -651,43 +651,40 @@ aosp_repo_modifications() { log_header ${FUNCNAME} cd "${BUILD_DIR}" - # make modifications to default AOSP - if ! grep -q "RattlesnakeOS" .repo/manifest.xml; then - # really ugly awk script to add additional repos to manifest - awk -i inplace \ - -v ANDROID_VERSION="$ANDROID_VERSION" \ - -v FDROID_CLIENT_VERSION="$FDROID_CLIENT_VERSION" \ - -v FDROID_PRIV_EXT_VERSION="$FDROID_PRIV_EXT_VERSION" \ - '1;/"; - print " "; - <% if .CustomManifestRemotes %> - <% range $i, $r := .CustomManifestRemotes %> - print " \" fetch=\"<% .Fetch %>\" revision=\"<% .Revision %>\" />"; - <% end %> - <% end %> - print " "; - <% if .CustomManifestProjects %><% range $i, $r := .CustomManifestProjects %> - print " \" name=\"<% .Name %>\" remote=\"<% .Remote %>\" />"; - <% end %> - <% end %> - <% if .EnableAttestation %> - print " "; - <% end %> - print " "; - print " "; - print " "; - print " "; - print " "}' .repo/manifest.xml - - # remove things from manifest - sed -i '/packages\/apps\/Browser2/d' .repo/manifest.xml - sed -i '/packages\/apps\/Calendar/d' .repo/manifest.xml - sed -i '/packages\/apps\/QuickSearchBox/d' .repo/manifest.xml - else - log "Skipping modification of .repo/manifest.xml as they have already been made" - fi + mkdir -p ${BUILD_DIR}/.repo/local_manifests + + cat < ${BUILD_DIR}/.repo/local_manifests/rattlesnakeos.xml + + + + + + + + + + + + + + + + <% if .CustomManifestRemotes %> + <% range $i, $r := .CustomManifestRemotes %> + + <% end %> + <% end %> + <% if .CustomManifestProjects %><% range $i, $r := .CustomManifestProjects %> + + <% end %> + <% end %> + <% if .EnableAttestation %> + + <% end %> + + +EOF + } aosp_repo_sync() { From e6ce7eba02f3a8ac2c013e807aa4f62df9daba8c Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Mon, 2 Mar 2020 21:55:11 -0800 Subject: [PATCH 10/23] workaround for upstream build issue with missing apifinder: https://issuetracker.google.com/issues/150626837 --- VERSION | 2 +- templates/build_template.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6b48f258..2681b301 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.4 +10.0.5 diff --git a/templates/build_template.go b/templates/build_template.go index 351fa32f..8aaa8119 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -653,6 +653,12 @@ aosp_repo_modifications() { mkdir -p ${BUILD_DIR}/.repo/local_manifests + # TODO: temporary fix for missing apifinder until upstream issue is resolved: https://issuetracker.google.com/issues/150626837 + temporary_apifinder_fix="" + if ! grep -q "platform/tools/apifinder" ${BUILD_DIR}/.repo/manifests/default.xml; then + temporary_apifinder_fix='' + fi + cat < ${BUILD_DIR}/.repo/local_manifests/rattlesnakeos.xml @@ -665,6 +671,8 @@ aosp_repo_modifications() { + $temporary_apifinder_fix + From c92182f3fa9b1e9d970256a36bb0545f45f20eaa Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Mon, 9 Mar 2020 12:50:41 -0700 Subject: [PATCH 11/23] * fix issue with sargo/bonito not picking up march update. add more elaborate search for latest correct factory build id. * no longer rely on https://source.android.com/setup/start/build-numbers for finding AOSP branch. instead just look at latest tags to find the one with matching factory build number. --- VERSION | 2 +- templates/build_template.go | 82 ++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/VERSION b/VERSION index 2681b301..80f86ac0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.5 +10.0.6 diff --git a/templates/build_template.go b/templates/build_template.go index 8aaa8119..b1196871 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -126,7 +126,7 @@ FDROID_CLIENT_URL_LATEST="https://gitlab.com/api/v4/projects/36189/repository/ta FDROID_PRIV_EXT_URL_LATEST="https://gitlab.com/api/v4/projects/1481578/repository/tags" KERNEL_SOURCE_URL="https://android.googlesource.com/kernel/msm" AOSP_URL_BUILD="https://developers.google.com/android/images" -AOSP_URL_BRANCH="https://source.android.com/setup/start/build-numbers" +AOSP_URL_PLATFORM_BUILD="https://android.googlesource.com/platform/build" STACK_UPDATE_MESSAGE= LATEST_STACK_VERSION= @@ -169,21 +169,81 @@ get_latest_versions() { exit 1 fi - # attempt to automatically pick latest build version and branch. note this is likely to break with any page redesign. should also add some validation here. + # check for latest factory image and corresponding aosp branch + # this is going to continue being quite fragile unless another method for determining latest factory build is found + # parsing html that can change at any time and can have variations every month is never going to be reliable + echo "Searching for latest factory build date for device=${DEVICE} android_version=${ANDROID_VERSION}" + latest_factory_build_date=$(curl --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | tail -1 | cut -d"(" -f2 | cut -d"," -f2 | cut -d")" -f1 | sed -e 's/^[ \t]*//' || true) + if [ -z "$latest_factory_build_date" ]; then + aws_notify_simple "ERROR: Unable to determine latest factory build date for device=${DEVICE} android_version=${ANDROID_VERSION}. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" + exit 1 + fi + echo "latest_factory_build_date='${latest_factory_build_date}'" + + # first check to see if any factory builds exist for this date + factory_builds=$(curl --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | grep "${latest_factory_build_date}" || true) + factory_builds_count=$(echo "$factory_builds" | wc -l | awk '{print $1}' || true) + if [ -z "$factory_builds" ]; then + factory_builds_count=0 + fi + if [ "$factory_builds_count" -eq 0 ]; then + aws_notify_simple "ERROR: Unable to find any builds for latest_factory_build_date='${latest_factory_build_date}'. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" + exit 1 + fi + + # if more than one factory build exists, + # first check strict (e.g. (QQ1A.200205.002, Feb 2020)) + # second check for known variations that Google has used in the past - for now just removing anything with 'only'" + if [ "$factory_builds_count" -ne 1 ]; then + echo -e "\nAttempting strict filter to a single factory build" + factory_builds_strict=$(echo "$factory_builds" | egrep '[A-Z]{1}[a-z]{2} [0-9]{4}\)' || true) + factory_builds_strict_count=$(echo "$factory_builds_strict" | wc -l | awk '{print $1}' || true) + if [ -z "$factory_builds_strict" ]; then + factory_builds_strict_count=0 + fi + + if [ "$factory_builds_strict_count" -ne 1 ]; then + echo -e "\nAttempting loose filter to a single factory build" + factory_builds_loose=$(echo "$factory_builds" | grep -i -v 'only' || true) + factory_builds_loose_count=$(echo "$factory_builds_loose" | wc -l | awk '{print $1}' || true) + if [ -z "$factory_builds_loose" ]; then + factory_builds_loose_count=0 + fi + if [ "$factory_builds_loose_count" -ne 1 ]; then + aws_notify_simple "ERROR: Unable to determine what build to use for latest_factory_build_date='${latest_factory_build_date}'. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" + echo "factory_builds_loose_count=${factory_builds_loose_count} factory_builds_loose=${factory_builds_loose}" + exit 1 + fi + factory_builds="${factory_builds_loose}" + else + factory_builds="${factory_builds_strict}" + fi + fi + AOSP_BUILD=$(echo "$factory_builds" | cut -d"(" -f2 | cut -d"," -f1 || true) if [ -z "$AOSP_BUILD" ]; then - AOSP_BUILD=$(curl --fail -s ${AOSP_URL_BUILD} | grep -A1 "${DEVICE}" | egrep '[a-zA-Z]+ [0-9]{4}\)' | grep -F "${ANDROID_VERSION}" | tail -1 | cut -d"(" -f2 | cut -d"," -f1) - if [ -z "$AOSP_BUILD" ]; then - aws_notify_simple "ERROR: Unable to get latest AOSP build information. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" + echo "Unable to determine factory build number when parsing factory_builds='${factory_builds}'" + aws_notify_simple "ERROR: Unable to get latest AOSP build (factory build) information. Stopping build." exit 1 - fi fi + echo "AOSP_BUILD=${AOSP_BUILD}" + + AOSP_BRANCH="" + echo "Searching through latest AOSP tags to find tag that corresponds with factory build ${AOSP_BRANCH}" + for tag in $(git ls-remote --tags "${AOSP_URL_PLATFORM_BUILD}" | grep "android-${ANDROID_VERSION}" | grep -v '\^{}' | awk -F"/" '{print $3}' | sort -V | tac || true); do + echo "Searching tag ${tag} for build_id=${AOSP_BUILD}" + build_id=$(curl -s "${AOSP_URL_PLATFORM_BUILD}/+/refs/tags/${tag}/core/build_id.mk?format=TEXT" | base64 --decode | awk -F= '/BUILD_ID=/{print $2}' || true) + if [ "${AOSP_BUILD}" = "${build_id}" ]; then + AOSP_BRANCH="${tag}" + break + fi + echo "Skipping tag ${tag} as build_id=${build_id}" + done if [ -z "$AOSP_BRANCH" ]; then - AOSP_BRANCH=$(curl --fail -s ${AOSP_URL_BRANCH} | grep -A1 "${AOSP_BUILD}" | tail -1 | cut -f2 -d">"|cut -f1 -d"<") - if [ -z "$AOSP_BRANCH" ]; then - aws_notify_simple "ERROR: Unable to get latest AOSP branch information. Stopping build. This can happen if ${AOSP_URL_BRANCH} hasn't been updated yet with newly released factory images." - exit 1 - fi + aws_notify_simple "ERROR: Unable to get latest AOSP branch information. Stopping build." + exit 1 fi + echo "AOSP_BRANCH=${AOSP_BRANCH}" + } check_for_new_versions() { From 88738688474e10e880f3b78626572b9b69309489 Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Mon, 9 Mar 2020 13:02:33 -0700 Subject: [PATCH 12/23] remove apifinder workaround now that upstream issue is resolved (https://issuetracker.google.com/issues/150626837) --- templates/build_template.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index b1196871..3aba6d7b 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -713,12 +713,6 @@ aosp_repo_modifications() { mkdir -p ${BUILD_DIR}/.repo/local_manifests - # TODO: temporary fix for missing apifinder until upstream issue is resolved: https://issuetracker.google.com/issues/150626837 - temporary_apifinder_fix="" - if ! grep -q "platform/tools/apifinder" ${BUILD_DIR}/.repo/manifests/default.xml; then - temporary_apifinder_fix='' - fi - cat < ${BUILD_DIR}/.repo/local_manifests/rattlesnakeos.xml @@ -731,8 +725,6 @@ aosp_repo_modifications() { - $temporary_apifinder_fix - From a62514c7714351daf9ecf3d0c7df4aa676a2deb1 Mon Sep 17 00:00:00 2001 From: Prasun Gera Date: Mon, 23 Mar 2020 13:31:42 -0400 Subject: [PATCH 13/23] Add support for building kernels for all devices Currently, prebuilt kernel images are used for most devices. There was code in place for marlin to rebuild the kernel. This extends the same flow to all devices. The feature is hidden behind the ENABLE_KERNEL_BUILD flag. After sufficient testing, it can be hooked to a config option. Eventually, this would also make it easier to support kernel patches. --- templates/build_template.go | 85 +++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index 3aba6d7b..6146d56b 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -13,29 +13,41 @@ DEVICE=$1 case "$DEVICE" in marlin|sailfish) DEVICE_FAMILY=marlin + KERNEL_FAMILY=marlin + KERNEL_DEFCONFIG=marlin AVB_MODE=verity_only ;; taimen) DEVICE_FAMILY=taimen + KERNEL_FAMILY=wahoo + KERNEL_DEFCONFIG=wahoo AVB_MODE=vbmeta_simple ;; walleye) DEVICE_FAMILY=muskie + KERNEL_FAMILY=wahoo + KERNEL_DEFCONFIG=wahoo AVB_MODE=vbmeta_simple ;; crosshatch|blueline) DEVICE_FAMILY=crosshatch + KERNEL_FAMILY=crosshatch + KERNEL_DEFCONFIG=b1c1 AVB_MODE=vbmeta_chained EXTRA_OTA=(--retrofit_dynamic_partitions) ;; sargo|bonito) DEVICE_FAMILY=bonito + KERNEL_FAMILY=bonito + KERNEL_DEFCONFIG=bonito AVB_MODE=vbmeta_chained EXTRA_OTA=(--retrofit_dynamic_partitions) ;; *) echo "warning: unknown device $DEVICE, using Pixel 3 defaults" DEVICE_FAMILY=$1 + KERNEL_FAMILY=crosshatch + KERNEL_DEFCONFIG=b1c1 AVB_MODE=vbmeta_chained ;; esac @@ -70,6 +82,10 @@ IGNORE_VERSION_CHECKS=<% .IgnoreVersionChecks %> # version of chromium to pin to if requested CHROMIUM_PINNED_VERSION=<% .ChromiumVersion %> +# Whether the kernel needs to be rebuilt +# It is always rebuilt for marlin/sailfish +ENABLE_KERNEL_BUILD=false + # whether keys are client side encrypted or not ENCRYPTED_KEYS="<% .EncryptedKeys %>" ENCRYPTION_KEY= @@ -114,7 +130,7 @@ BUILD_DIR="$HOME/rattlesnake-os" KEYS_DIR="${BUILD_DIR}/keys" CERTIFICATE_SUBJECT='/CN=RattlesnakeOS' OFFICIAL_FDROID_KEY="43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab" -MARLIN_KERNEL_SOURCE_DIR="${HOME}/kernel/google/marlin" +KERNEL_SOURCE_DIR="${HOME}/kernel/google/${KERNEL_FAMILY}" BUILD_REASON="" # urls @@ -354,8 +370,9 @@ full_run() { build_fdroid apply_patches # only marlin and sailfish need kernel rebuilt so that verity_key is included - if [ "${DEVICE}" == "marlin" ] || [ "${DEVICE}" == "sailfish" ]; then - rebuild_marlin_kernel + # Also build the kernel if enabled in the config + if [ "${KERNEL_FAMILY}" == "marlin" ] || [ "${ENABLE_KERNEL_BUILD}" == "true" ]; then + rebuild_kernel fi add_chromium build_aosp @@ -996,18 +1013,27 @@ patch_launcher() { sed -i.original "s/boolean createEmptyRowOnFirstScreen;/boolean createEmptyRowOnFirstScreen = false;/" "${BUILD_DIR}/packages/apps/Launcher3/src/com/android/launcher3/provider/ImportDataTask.java" } -rebuild_marlin_kernel() { +rebuild_kernel() { log_header ${FUNCNAME} # checkout kernel source on proper commit - mkdir -p "${MARLIN_KERNEL_SOURCE_DIR}" - retry git clone "${KERNEL_SOURCE_URL}" "${MARLIN_KERNEL_SOURCE_DIR}" + mkdir -p "${KERNEL_SOURCE_DIR}" + retry git clone "${KERNEL_SOURCE_URL}" "${KERNEL_SOURCE_DIR}" + + if [ "${KERNEL_FAMILY}" == "marlin" ] || [ "${KERNEL_FAMILY}" == "wahoo" ]; then + kernel_image="${BUILD_DIR}/device/google/${KERNEL_FAMILY}-kernel/Image.lz4-dtb" + else + kernel_image="${BUILD_DIR}/device/google/${KERNEL_FAMILY}-kernel/Image.lz4" + fi + # TODO: make this a bit more robust - kernel_commit_id=$(lz4cat "${BUILD_DIR}/device/google/marlin-kernel/Image.lz4-dtb" | grep -a 'Linux version' | cut -d ' ' -f3 | cut -d'-' -f2 | sed 's/^g//g') - cd "${MARLIN_KERNEL_SOURCE_DIR}" + kernel_commit_id=$(lz4cat "${kernel_image}" | strings | grep -a 'Linux version [0-9]' | cut -d ' ' -f3 | cut -d'-' -f2 | sed 's/^g//g') + cd "${KERNEL_SOURCE_DIR}" log "Checking out kernel commit ${kernel_commit_id}" git checkout ${kernel_commit_id} + # TODO: kernel patch hooks should be added here + # run in another shell to avoid it mucking with environment variables for normal AOSP build ( set -e; @@ -1016,11 +1042,44 @@ rebuild_marlin_kernel() { export PATH="${BUILD_DIR}/prebuilts/misc/linux-x86/lz4:${PATH}"; export PATH="${BUILD_DIR}/prebuilts/misc/linux-x86/dtc:${PATH}"; export PATH="${BUILD_DIR}/prebuilts/misc/linux-x86/libufdt:${PATH}"; - ln --verbose --symbolic ${KEYS_DIR}/${DEVICE}/verity_user.der.x509 ${MARLIN_KERNEL_SOURCE_DIR}/verity_user.der.x509; - cd ${MARLIN_KERNEL_SOURCE_DIR}; - make O=out ARCH=arm64 marlin_defconfig; - make -j$(nproc --all) O=out ARCH=arm64 CROSS_COMPILE=aarch64-linux-android- CROSS_COMPILE_ARM32=arm-linux-androideabi- - cp -f out/arch/arm64/boot/Image.lz4-dtb ${BUILD_DIR}/device/google/marlin-kernel/; + cd ${KERNEL_SOURCE_DIR}; + + if [ "${KERNEL_FAMILY}" == "marlin" ]; then + ln --verbose --symbolic ${KEYS_DIR}/${DEVICE}/verity_user.der.x509 ${KERNEL_SOURCE_DIR}/verity_user.der.x509; + make O=out ARCH=arm64 ${KERNEL_DEFCONFIG}_defconfig; + make -j$(nproc --all) \ + O=out \ + ARCH=arm64 \ + CROSS_COMPILE=aarch64-linux-android- \ + CROSS_COMPILE_ARM32=arm-linux-androideabi- + + cp -f out/arch/arm64/boot/Image.lz4-dtb ${BUILD_DIR}/device/google/${KERNEL_FAMILY}-kernel/; + fi + + if [ "${KERNEL_FAMILY}" == "wahoo" ] || [ "${KERNEL_FAMILY}" == "crosshatch" ] || [ "${KERNEL_FAMILY}" == "bonito" ]; then + export PATH="${BUILD_DIR}/prebuilts/clang/host/linux-x86/clang-r353983c/bin:${PATH}"; + export LD_LIBRARY_PATH="${BUILD_DIR}/prebuilts/clang/host/linux-x86/clang-r353983c/lib64:${LD_LIBRARY_PATH}"; + make O=out ARCH=arm64 ${KERNEL_DEFCONFIG}_defconfig; + make -j$(nproc --all) \ + O=out \ + ARCH=arm64 \ + CC=clang \ + CLANG_TRIPLE=aarch64-linux-gnu- \ + CROSS_COMPILE=aarch64-linux-android- \ + CROSS_COMPILE_ARM32=arm-linux-androideabi- + + cp -f out/arch/arm64/boot/{dtbo.img,Image.lz4-dtb} ${BUILD_DIR}/device/google/${KERNEL_FAMILY}-kernel/; + + if [ "${KERNEL_FAMILY}" == "crosshatch" ]; then + cp -f out/arch/arm64/boot/dts/qcom/{sdm845-v2.dtb,sdm845-v2.1.dtb} ${BUILD_DIR}/device/google/${KERNEL_FAMILY}-kernel/; + fi + + if [ "${KERNEL_FAMILY}" == "bonito" ]; then + cp -f out/arch/arm64/boot/dts/qcom/sdm670.dtb ${BUILD_DIR}/device/google/${KERNEL_FAMILY}-kernel/; + fi + + fi + rm -rf ${BUILD_DIR}/out/build_*; ) } From b225e1c70f7bc114bfc45335fef2be1aa192ecce Mon Sep 17 00:00:00 2001 From: khimaros Date: Thu, 19 Mar 2020 19:42:45 -0700 Subject: [PATCH 14/23] allow builds with release branches which are newer than the latest factory image This change should be fully backward compatible and will enable users to build images from the latest release branches for sailfish and marlin devices, even though there are no current factory images. The specific changes are: 1) decouple `AOSP_BUILD` from the automated vendor image search and 2) when the AOSP_BUILD and factory versions differ, write the provided AOSP_BUILD to `build_id.txt` prior to starting the AOSP build. The diff on lines 230 through 251 are largely the result of indentation. --- templates/build_template.go | 62 +++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index 6146d56b..71b62c3a 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -62,6 +62,7 @@ fi # allow build and branch to be specified AOSP_BUILD=$3 AOSP_BRANCH=$4 +AOSP_VENDOR_BUILD= # set region REGION=<% .Region %> @@ -235,28 +236,35 @@ get_latest_versions() { factory_builds="${factory_builds_strict}" fi fi - AOSP_BUILD=$(echo "$factory_builds" | cut -d"(" -f2 | cut -d"," -f1 || true) - if [ -z "$AOSP_BUILD" ]; then + + AOSP_VENDOR_BUILD=$(echo "$factory_builds" | cut -d"(" -f2 | cut -d"," -f1 || true) + if [ -z "$AOSP_VENDOR_BUILD" ]; then echo "Unable to determine factory build number when parsing factory_builds='${factory_builds}'" aws_notify_simple "ERROR: Unable to get latest AOSP build (factory build) information. Stopping build." exit 1 fi + if [ -z "$AOSP_BUILD" ]; then + AOSP_BUILD=${AOSP_VENDOR_BUILD} + fi + echo "AOSP_VENDOR_BUILD=${AOSP_VENDOR_BUILD}" echo "AOSP_BUILD=${AOSP_BUILD}" - AOSP_BRANCH="" - echo "Searching through latest AOSP tags to find tag that corresponds with factory build ${AOSP_BRANCH}" - for tag in $(git ls-remote --tags "${AOSP_URL_PLATFORM_BUILD}" | grep "android-${ANDROID_VERSION}" | grep -v '\^{}' | awk -F"/" '{print $3}' | sort -V | tac || true); do - echo "Searching tag ${tag} for build_id=${AOSP_BUILD}" - build_id=$(curl -s "${AOSP_URL_PLATFORM_BUILD}/+/refs/tags/${tag}/core/build_id.mk?format=TEXT" | base64 --decode | awk -F= '/BUILD_ID=/{print $2}' || true) - if [ "${AOSP_BUILD}" = "${build_id}" ]; then - AOSP_BRANCH="${tag}" - break - fi - echo "Skipping tag ${tag} as build_id=${build_id}" - done if [ -z "$AOSP_BRANCH" ]; then - aws_notify_simple "ERROR: Unable to get latest AOSP branch information. Stopping build." - exit 1 + AOSP_BRANCH="" + echo "Searching through latest AOSP tags to find tag that corresponds with factory build ${AOSP_BRANCH}" + for tag in $(git ls-remote --tags "${AOSP_URL_PLATFORM_BUILD}" | grep "android-${ANDROID_VERSION}" | grep -v '\^{}' | awk -F"/" '{print $3}' | sort -V | tac || true); do + echo "Searching tag ${tag} for build_id=${AOSP_BUILD}" + build_id=$(curl -s "${AOSP_URL_PLATFORM_BUILD}/+/refs/tags/${tag}/core/build_id.mk?format=TEXT" | base64 --decode | awk -F= '/BUILD_ID=/{print $2}' || true) + if [ "${AOSP_BUILD}" = "${build_id}" ]; then + AOSP_BRANCH="${tag}" + break + fi + echo "Skipping tag ${tag} as build_id=${build_id}" + done + if [ -z "$AOSP_BRANCH" ]; then + aws_notify_simple "ERROR: Unable to get latest AOSP branch information. Stopping build." + exit 1 + fi fi echo "AOSP_BRANCH=${AOSP_BRANCH}" @@ -280,12 +288,12 @@ check_for_new_versions() { # check aosp existing_aosp_build=$(aws s3 cp "s3://${AWS_RELEASE_BUCKET}/${DEVICE}-vendor" - || true) - if [ "$existing_aosp_build" == "$AOSP_BUILD" ]; then + if [ "$existing_aosp_build" == "$AOSP_VENDOR_BUILD" ]; then echo "AOSP build ($existing_aosp_build) is up to date" else - echo "AOSP needs to be updated to ${AOSP_BUILD}" + echo "AOSP needs to be updated to ${AOSP_VENDOR_BUILD}" needs_update=true - BUILD_REASON="$BUILD_REASON 'AOSP build $existing_aosp_build != $AOSP_BUILD'" + BUILD_REASON="$BUILD_REASON 'AOSP build $existing_aosp_build != $AOSP_VENDOR_BUILD'" fi # check chromium @@ -781,17 +789,17 @@ setup_vendor() { sudo DEBIAN_FRONTEND=noninteractive apt-get -y install python-protobuf # get vendor files (with timeout) - timeout 30m "${BUILD_DIR}/vendor/android-prepare-vendor/execute-all.sh" --debugfs --keep --yes --device "${DEVICE}" --buildID "${AOSP_BUILD}" --output "${BUILD_DIR}/vendor/android-prepare-vendor" + timeout 30m "${BUILD_DIR}/vendor/android-prepare-vendor/execute-all.sh" --debugfs --keep --yes --device "${DEVICE}" --buildID "${AOSP_VENDOR_BUILD}" --output "${BUILD_DIR}/vendor/android-prepare-vendor" # copy vendor files to build tree mkdir --parents "${BUILD_DIR}/vendor/google_devices" || true rm -rf "${BUILD_DIR}/vendor/google_devices/$DEVICE" || true - mv "${BUILD_DIR}/vendor/android-prepare-vendor/${DEVICE}/$(tr '[:upper:]' '[:lower:]' <<< "${AOSP_BUILD}")/vendor/google_devices/${DEVICE}" "${BUILD_DIR}/vendor/google_devices" + mv "${BUILD_DIR}/vendor/android-prepare-vendor/${DEVICE}/$(tr '[:upper:]' '[:lower:]' <<< "${AOSP_VENDOR_BUILD}")/vendor/google_devices/${DEVICE}" "${BUILD_DIR}/vendor/google_devices" # smaller devices need big brother vendor files if [ "$DEVICE" != "$DEVICE_FAMILY" ]; then rm -rf "${BUILD_DIR}/vendor/google_devices/$DEVICE_FAMILY" || true - mv "${BUILD_DIR}/vendor/android-prepare-vendor/$DEVICE/$(tr '[:upper:]' '[:lower:]' <<< "${AOSP_BUILD}")/vendor/google_devices/$DEVICE_FAMILY" "${BUILD_DIR}/vendor/google_devices" + mv "${BUILD_DIR}/vendor/android-prepare-vendor/$DEVICE/$(tr '[:upper:]' '[:lower:]' <<< "${AOSP_VENDOR_BUILD}")/vendor/google_devices/$DEVICE_FAMILY" "${BUILD_DIR}/vendor/google_devices" fi } @@ -1089,6 +1097,12 @@ build_aosp() { cd "$BUILD_DIR" + if [ "${AOSP_BUILD}" != "${AOSP_VENDOR_BUILD}" ]; then + log "WARNING: Requested AOSP build does not match upstream vendor files. These images may not be functional." + log "Patching build_id to match ${AOSP_BUILD}" + echo "${AOSP_BUILD}" > vendor/google_devices/${DEVICE}/build_id.txt + fi + ############################ # from original setup.sh script ############################ @@ -1246,7 +1260,7 @@ checkpoint_versions() { echo "${FDROID_CLIENT_VERSION}" | aws s3 cp - "s3://${AWS_RELEASE_BUCKET}/fdroid/revision" # checkpoint aosp - aws s3 cp - "s3://${AWS_RELEASE_BUCKET}/${DEVICE}-vendor" --acl public-read <<< "${AOSP_BUILD}" || true + aws s3 cp - "s3://${AWS_RELEASE_BUCKET}/${DEVICE}-vendor" --acl public-read <<< "${AOSP_VENDOR_BUILD}" || true # checkpoint chromium echo "yes" | aws s3 cp - "s3://${AWS_RELEASE_BUCKET}/chromium/included" @@ -1267,8 +1281,8 @@ aws_notify() { fi ELAPSED="$(($SECONDS / 3600))hrs $((($SECONDS / 60) % 60))min $(($SECONDS % 60))sec" aws sns publish --region ${REGION} --topic-arn "$AWS_SNS_ARN" \ - --message="$(printf "$1\n Device: %s\n Stack Name: %s\n Stack Version: %s %s\n Stack Region: %s\n Release Channel: %s\n Instance Type: %s\n Instance Region: %s\n Instance IP: %s\n Build Date: %s\n Elapsed Time: %s\n AOSP Build: %s\n AOSP Branch: %s\n Chromium Version: %s\n F-Droid Version: %s\n F-Droid Priv Extension Version: %s\n Build Reason: %s\n%s" \ - "${DEVICE}" "${STACK_NAME}" "${STACK_VERSION}" "${STACK_UPDATE_MESSAGE}" "${REGION}" "${RELEASE_CHANNEL}" "${INSTANCE_TYPE}" "${INSTANCE_REGION}" "${INSTANCE_IP}" "${BUILD_DATE}" "${ELAPSED}" "${AOSP_BUILD}" "${AOSP_BRANCH}" "${LATEST_CHROMIUM}" "${FDROID_CLIENT_VERSION}" "${FDROID_PRIV_EXT_VERSION}" "${BUILD_REASON}" "${LOGOUTPUT}")" || true + --message="$(printf "$1\n Device: %s\n Stack Name: %s\n Stack Version: %s %s\n Stack Region: %s\n Release Channel: %s\n Instance Type: %s\n Instance Region: %s\n Instance IP: %s\n Build Date: %s\n Elapsed Time: %s\n AOSP Build: %s\n AOSP Vendor Build: %s\n AOSP Branch: %s\n Chromium Version: %s\n F-Droid Version: %s\n F-Droid Priv Extension Version: %s\n Build Reason: %s\n%s" \ + "${DEVICE}" "${STACK_NAME}" "${STACK_VERSION}" "${STACK_UPDATE_MESSAGE}" "${REGION}" "${RELEASE_CHANNEL}" "${INSTANCE_TYPE}" "${INSTANCE_REGION}" "${INSTANCE_IP}" "${BUILD_DATE}" "${ELAPSED}" "${AOSP_BUILD}" "${AOSP_VENDOR_BUILD}" "${AOSP_BRANCH}" "${LATEST_CHROMIUM}" "${FDROID_CLIENT_VERSION}" "${FDROID_PRIV_EXT_VERSION}" "${BUILD_REASON}" "${LOGOUTPUT}")" || true } aws_logging() { From e95d4ea71ac05f8374c9eb0e93447f092c298288 Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Mon, 4 May 2020 13:54:50 -0700 Subject: [PATCH 15/23] additional duct tape for may 2020 factory build parsing issue (yet again) --- VERSION | 2 +- templates/build_template.go | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 80f86ac0..9380cfcc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.6 +10.0.7 diff --git a/templates/build_template.go b/templates/build_template.go index 71b62c3a..a3a708e6 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -211,6 +211,7 @@ get_latest_versions() { # if more than one factory build exists, # first check strict (e.g. (QQ1A.200205.002, Feb 2020)) # second check for known variations that Google has used in the past - for now just removing anything with 'only'" + # this is fragile and likely need to move to a file tracking this instead: https://github.com/dan-v/rattlesnakeos-stack/issues/153 if [ "$factory_builds_count" -ne 1 ]; then echo -e "\nAttempting strict filter to a single factory build" factory_builds_strict=$(echo "$factory_builds" | egrep '[A-Z]{1}[a-z]{2} [0-9]{4}\)' || true) @@ -223,8 +224,12 @@ get_latest_versions() { echo -e "\nAttempting loose filter to a single factory build" factory_builds_loose=$(echo "$factory_builds" | grep -i -v 'only' || true) factory_builds_loose_count=$(echo "$factory_builds_loose" | wc -l | awk '{print $1}' || true) - if [ -z "$factory_builds_loose" ]; then - factory_builds_loose_count=0 + if [ -z "$factory_builds_loose" ] || [ "$factory_builds_loose_count" -gt 1 ]; then + factory_builds_loose=$(echo "$factory_builds" | grep -i 'All carriers except' || true) + factory_builds_loose_count=$(echo "$factory_builds_loose" | wc -l | awk '{print $1}' || true) + if [ -z "$factory_builds_loose" ]; then + $factory_builds_loose_count=0 + fi fi if [ "$factory_builds_loose_count" -ne 1 ]; then aws_notify_simple "ERROR: Unable to determine what build to use for latest_factory_build_date='${latest_factory_build_date}'. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" From 42d90528c18b49e738615e240f0fde54110f9ffc Mon Sep 17 00:00:00 2001 From: Scott <13770321+scott-8@users.noreply.github.com> Date: Sun, 10 May 2020 00:35:28 -0400 Subject: [PATCH 16/23] Fix failed parse of images site --- templates/build_template.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index a3a708e6..360af56f 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -190,7 +190,7 @@ get_latest_versions() { # this is going to continue being quite fragile unless another method for determining latest factory build is found # parsing html that can change at any time and can have variations every month is never going to be reliable echo "Searching for latest factory build date for device=${DEVICE} android_version=${ANDROID_VERSION}" - latest_factory_build_date=$(curl --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | tail -1 | cut -d"(" -f2 | cut -d"," -f2 | cut -d")" -f1 | sed -e 's/^[ \t]*//' || true) + latest_factory_build_date=$(curl --cookie "devsite_wall_acks=nexus-image-tos" --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | tail -1 | cut -d"(" -f2 | cut -d"," -f2 | cut -d")" -f1 | sed -e 's/^[ \t]*//' || true) if [ -z "$latest_factory_build_date" ]; then aws_notify_simple "ERROR: Unable to determine latest factory build date for device=${DEVICE} android_version=${ANDROID_VERSION}. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" exit 1 @@ -198,7 +198,7 @@ get_latest_versions() { echo "latest_factory_build_date='${latest_factory_build_date}'" # first check to see if any factory builds exist for this date - factory_builds=$(curl --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | grep "${latest_factory_build_date}" || true) + factory_builds=$(curl --cookie "devsite_wall_acks=nexus-image-tos" --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | grep "${latest_factory_build_date}" || true) factory_builds_count=$(echo "$factory_builds" | wc -l | awk '{print $1}' || true) if [ -z "$factory_builds" ]; then factory_builds_count=0 From ea1e7c38a0603458566e98ba0218d479b35b24cf Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sat, 9 May 2020 23:47:31 -0700 Subject: [PATCH 17/23] change over to using https://github.com/RattlesnakeOS/latest to lookup latest aosp build/tag, chromium, fdroid versions. closes #153 --- VERSION | 2 +- templates/build_template.go | 108 +++++++----------------------------- 2 files changed, 22 insertions(+), 88 deletions(-) diff --git a/VERSION b/VERSION index 9380cfcc..5219a0df 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.7 +10.0.8 diff --git a/templates/build_template.go b/templates/build_template.go index 360af56f..92c55a88 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -137,13 +137,14 @@ BUILD_REASON="" # urls ANDROID_SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip" MANIFEST_URL="https://android.googlesource.com/platform/manifest" -CHROME_URL_LATEST="https://omahaproxy.appspot.com/all.json" STACK_URL_LATEST="https://api.github.com/repos/dan-v/rattlesnakeos-stack/releases/latest" -FDROID_CLIENT_URL_LATEST="https://gitlab.com/api/v4/projects/36189/repository/tags" -FDROID_PRIV_EXT_URL_LATEST="https://gitlab.com/api/v4/projects/1481578/repository/tags" KERNEL_SOURCE_URL="https://android.googlesource.com/kernel/msm" AOSP_URL_BUILD="https://developers.google.com/android/images" AOSP_URL_PLATFORM_BUILD="https://android.googlesource.com/platform/build" +RATTLESNAKEOS_LATEST_JSON="https://raw.githubusercontent.com/RattlesnakeOS/latest/${ANDROID_VERSION}" +RATTLESNAKEOS_LATEST_JSON_AOSP="${RATTLESNAKEOS_LATEST_JSON}/aosp.json" +RATTLESNAKEOS_LATEST_JSON_CHROMIUM="${RATTLESNAKEOS_LATEST_JSON}/chromium.json" +RATTLESNAKEOS_LATEST_JSON_FDROID="${RATTLESNAKEOS_LATEST_JSON}/fdroid.json" STACK_UPDATE_MESSAGE= LATEST_STACK_VERSION= @@ -166,110 +167,43 @@ get_latest_versions() { STACK_UPDATE_MESSAGE="WARNING: you should upgrade to the latest version: ${LATEST_STACK_VERSION}" fi - # check for latest stable chromium version - LATEST_CHROMIUM=$(curl --fail -s "$CHROME_URL_LATEST" | jq -r '.[] | select(.os == "android") | .versions[] | select(.channel == "'$CHROME_CHANNEL'") | .current_version') + # check for latest chromium version + LATEST_CHROMIUM=$(curl --fail -s "${RATTLESNAKEOS_LATEST_JSON_CHROMIUM}" | jq -r ".$CHROME_CHANNEL") if [ -z "$LATEST_CHROMIUM" ]; then aws_notify_simple "ERROR: Unable to get latest Chromium version details. Stopping build." exit 1 fi + echo "LATEST_CHROMIUM=${LATEST_CHROMIUM}" - # fdroid - get latest non alpha tags from gitlab (sorted) - # TODO: exclude alpha once 1.8 stable is released - FDROID_CLIENT_VERSION=$(curl --fail -s "$FDROID_CLIENT_URL_LATEST" | jq -r '[.[] | select(.name | test("^[0-9]+\\.[0-9]+")) | select(.name | contains("ota") | not)][] | .name' | sort --version-sort -r | head -1) + FDROID_CLIENT_VERSION=$(curl --fail -s "${RATTLESNAKEOS_LATEST_JSON_FDROID}" | jq -r ".client") if [ -z "$FDROID_CLIENT_VERSION" ]; then aws_notify_simple "ERROR: Unable to get latest F-Droid version details. Stopping build." exit 1 fi - FDROID_PRIV_EXT_VERSION=$(curl --fail -s "$FDROID_PRIV_EXT_URL_LATEST" | jq -r '[.[] | select(.name | test("^[0-9]+\\.[0-9]+")) | select(.name | contains("alpha") | not) | select(.name | contains("ota") | not)][] | .name' | sort --version-sort -r | head -1) + echo "FDROID_CLIENT_VERSION=${FDROID_CLIENT_VERSION}" + + FDROID_PRIV_EXT_VERSION=$(curl --fail -s "${RATTLESNAKEOS_LATEST_JSON_FDROID}" | jq -r ".privilegedextention") if [ -z "$FDROID_PRIV_EXT_VERSION" ]; then aws_notify_simple "ERROR: Unable to get latest F-Droid privilege extension version details. Stopping build." exit 1 fi + echo "FDROID_PRIV_EXT_VERSION=${FDROID_PRIV_EXT_VERSION}" - # check for latest factory image and corresponding aosp branch - # this is going to continue being quite fragile unless another method for determining latest factory build is found - # parsing html that can change at any time and can have variations every month is never going to be reliable - echo "Searching for latest factory build date for device=${DEVICE} android_version=${ANDROID_VERSION}" - latest_factory_build_date=$(curl --cookie "devsite_wall_acks=nexus-image-tos" --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | tail -1 | cut -d"(" -f2 | cut -d"," -f2 | cut -d")" -f1 | sed -e 's/^[ \t]*//' || true) - if [ -z "$latest_factory_build_date" ]; then - aws_notify_simple "ERROR: Unable to determine latest factory build date for device=${DEVICE} android_version=${ANDROID_VERSION}. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" - exit 1 - fi - echo "latest_factory_build_date='${latest_factory_build_date}'" - - # first check to see if any factory builds exist for this date - factory_builds=$(curl --cookie "devsite_wall_acks=nexus-image-tos" --fail -s "${AOSP_URL_BUILD}" | grep -A1 "${DEVICE}" | grep -F "${ANDROID_VERSION}" | grep "${latest_factory_build_date}" || true) - factory_builds_count=$(echo "$factory_builds" | wc -l | awk '{print $1}' || true) - if [ -z "$factory_builds" ]; then - factory_builds_count=0 - fi - if [ "$factory_builds_count" -eq 0 ]; then - aws_notify_simple "ERROR: Unable to find any builds for latest_factory_build_date='${latest_factory_build_date}'. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" - exit 1 - fi - - # if more than one factory build exists, - # first check strict (e.g. (QQ1A.200205.002, Feb 2020)) - # second check for known variations that Google has used in the past - for now just removing anything with 'only'" - # this is fragile and likely need to move to a file tracking this instead: https://github.com/dan-v/rattlesnakeos-stack/issues/153 - if [ "$factory_builds_count" -ne 1 ]; then - echo -e "\nAttempting strict filter to a single factory build" - factory_builds_strict=$(echo "$factory_builds" | egrep '[A-Z]{1}[a-z]{2} [0-9]{4}\)' || true) - factory_builds_strict_count=$(echo "$factory_builds_strict" | wc -l | awk '{print $1}' || true) - if [ -z "$factory_builds_strict" ]; then - factory_builds_strict_count=0 - fi - - if [ "$factory_builds_strict_count" -ne 1 ]; then - echo -e "\nAttempting loose filter to a single factory build" - factory_builds_loose=$(echo "$factory_builds" | grep -i -v 'only' || true) - factory_builds_loose_count=$(echo "$factory_builds_loose" | wc -l | awk '{print $1}' || true) - if [ -z "$factory_builds_loose" ] || [ "$factory_builds_loose_count" -gt 1 ]; then - factory_builds_loose=$(echo "$factory_builds" | grep -i 'All carriers except' || true) - factory_builds_loose_count=$(echo "$factory_builds_loose" | wc -l | awk '{print $1}' || true) - if [ -z "$factory_builds_loose" ]; then - $factory_builds_loose_count=0 - fi - fi - if [ "$factory_builds_loose_count" -ne 1 ]; then - aws_notify_simple "ERROR: Unable to determine what build to use for latest_factory_build_date='${latest_factory_build_date}'. Stopping build. This lookup is pretty fragile and can break on any page redesign of ${AOSP_URL_BUILD}" - echo "factory_builds_loose_count=${factory_builds_loose_count} factory_builds_loose=${factory_builds_loose}" - exit 1 - fi - factory_builds="${factory_builds_loose}" - else - factory_builds="${factory_builds_strict}" - fi - fi - - AOSP_VENDOR_BUILD=$(echo "$factory_builds" | cut -d"(" -f2 | cut -d"," -f1 || true) - if [ -z "$AOSP_VENDOR_BUILD" ]; then - echo "Unable to determine factory build number when parsing factory_builds='${factory_builds}'" - aws_notify_simple "ERROR: Unable to get latest AOSP build (factory build) information. Stopping build." - exit 1 + AOSP_VENDOR_BUILD=$(curl --fail -s "${RATTLESNAKEOS_LATEST_JSON_AOSP}" | jq -r ".${DEVICE}.build") + if [ -z "AOSP_VENDOR_BUILD" ]; then + aws_notify_simple "ERROR: Unable to get latest AOSP build version details. Stopping build." + exit 1 fi if [ -z "$AOSP_BUILD" ]; then - AOSP_BUILD=${AOSP_VENDOR_BUILD} + AOSP_BUILD=${AOSP_VENDOR_BUILD} fi echo "AOSP_VENDOR_BUILD=${AOSP_VENDOR_BUILD}" echo "AOSP_BUILD=${AOSP_BUILD}" - if [ -z "$AOSP_BRANCH" ]; then - AOSP_BRANCH="" - echo "Searching through latest AOSP tags to find tag that corresponds with factory build ${AOSP_BRANCH}" - for tag in $(git ls-remote --tags "${AOSP_URL_PLATFORM_BUILD}" | grep "android-${ANDROID_VERSION}" | grep -v '\^{}' | awk -F"/" '{print $3}' | sort -V | tac || true); do - echo "Searching tag ${tag} for build_id=${AOSP_BUILD}" - build_id=$(curl -s "${AOSP_URL_PLATFORM_BUILD}/+/refs/tags/${tag}/core/build_id.mk?format=TEXT" | base64 --decode | awk -F= '/BUILD_ID=/{print $2}' || true) - if [ "${AOSP_BUILD}" = "${build_id}" ]; then - AOSP_BRANCH="${tag}" - break - fi - echo "Skipping tag ${tag} as build_id=${build_id}" - done - if [ -z "$AOSP_BRANCH" ]; then - aws_notify_simple "ERROR: Unable to get latest AOSP branch information. Stopping build." - exit 1 - fi + AOSP_BRANCH=$(curl --fail -s "${RATTLESNAKEOS_LATEST_JSON_AOSP}" | jq -r ".${DEVICE}.branch") + if [ -z "AOSP_BUILD" ]; then + aws_notify_simple "ERROR: Unable to get latest AOSP branch details. Stopping build." + exit 1 fi echo "AOSP_BRANCH=${AOSP_BRANCH}" From 1342531cc453dbc683e2047d1955e3016dbd4a1d Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Sun, 10 May 2020 14:15:01 -0700 Subject: [PATCH 18/23] add retry to repo init --- templates/build_template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/build_template.go b/templates/build_template.go index 92c55a88..a7cfab03 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -668,7 +668,7 @@ aosp_repo_init() { log_header ${FUNCNAME} cd "${BUILD_DIR}" - repo init --manifest-url "$MANIFEST_URL" --manifest-branch "$AOSP_BRANCH" --depth 1 || true + retry repo init --manifest-url "$MANIFEST_URL" --manifest-branch "$AOSP_BRANCH" --depth 1 || true } aosp_repo_modifications() { From 9564f1506c208c242d5537a2d1864894657b97da Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Fri, 15 May 2020 15:02:21 -0700 Subject: [PATCH 19/23] add support for pixel 4 (coral/flame) --- README.md | 2 +- VERSION | 2 +- cli/deploy.go | 10 ++++++++-- templates/build_template.go | 20 +++++++++++++++++++- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d139e743..4317b51b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ RattlesnakeOS is a privacy and security focused Android OS for Google Pixel phon ## Features * Based on latest [AOSP](https://source.android.com/) 10.0 -* Active Support for Google Pixel 2, Pixel 2 XL, Pixel 3, Pixel 3 XL, Pixel 3a, Pixel 3a XL +* Active Support for Google Pixel 2-4(a) (XL) * Legacy Support for Google Pixel, Pixel XL. These devices no longer receive security updates and will eventually be deprecated. * Monthly software and firmware security fixes delivered through built in OTA updater * [Verified boot](https://source.android.com/security/verifiedboot/) with a locked bootloader just like official Android but with your own personal signing keys diff --git a/VERSION b/VERSION index 5219a0df..82f3d338 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.8 +10.0.9 diff --git a/cli/deploy.go b/cli/deploy.go index dae322cc..0f629634 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -32,8 +32,14 @@ var supportedRegions = []string{"ap-northeast-1", "ap-northeast-2", "ap-northeas "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2", "cn-northwest-1", "cn-north-1"} -var supportedDevicesFriendly = []string{"Pixel", "Pixel XL", "Pixel 2", "Pixel 2 XL", "Pixel 3", "Pixel 3 XL", "Pixel 3a", "Pixel 3a XL"} -var supportedDevicesCodename = []string{"sailfish", "marlin", "walleye", "taimen", "blueline", "crosshatch", "sargo", "bonito"} +var supportedDevicesFriendly = []string{ + "Pixel", "Pixel XL", "Pixel 2", "Pixel 2 XL", + "Pixel 3", "Pixel 3 XL", "Pixel 3a", "Pixel 3a XL", + "Pixel 4", "Pixel 4 XL"} +var supportedDevicesCodename = []string{ + "sailfish", "marlin", "walleye", "taimen", + "blueline", "crosshatch", "sargo", "bonito", + "flame", "coral"} var supportDevicesOutput string func init() { diff --git a/templates/build_template.go b/templates/build_template.go index a7cfab03..bc4ee593 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -43,6 +43,12 @@ case "$DEVICE" in AVB_MODE=vbmeta_chained EXTRA_OTA=(--retrofit_dynamic_partitions) ;; + flame|coral) + DEVICE_FAMILY=coral + KERNEL_FAMILY=coral + KERNEL_DEFCONFIG=coral + AVB_MODE=vbmeta_chained_v2 + ;; *) echo "warning: unknown device $DEVICE, using Pixel 3 defaults" DEVICE_FAMILY=$1 @@ -896,6 +902,9 @@ patch_device_config() { sed -i 's@PRODUCT_MODEL := AOSP on bonito@PRODUCT_MODEL := Pixel 3a XL@' ${BUILD_DIR}/device/google/bonito/aosp_bonito.mk || true sed -i 's@PRODUCT_MODEL := AOSP on sargo@PRODUCT_MODEL := Pixel 3a@' ${BUILD_DIR}/device/google/bonito/aosp_sargo.mk || true + + sed -i 's@PRODUCT_MODEL := AOSP on coral@PRODUCT_MODEL := Pixel 4 XL@' ${BUILD_DIR}/device/google/coral/aosp_coral.mk || true + sed -i 's@PRODUCT_MODEL := AOSP on flame@PRODUCT_MODEL := Pixel 4@' ${BUILD_DIR}/device/google/coral/aosp_flame.mk || true } get_package_mk_file() { @@ -1003,7 +1012,8 @@ rebuild_kernel() { cp -f out/arch/arm64/boot/Image.lz4-dtb ${BUILD_DIR}/device/google/${KERNEL_FAMILY}-kernel/; fi - if [ "${KERNEL_FAMILY}" == "wahoo" ] || [ "${KERNEL_FAMILY}" == "crosshatch" ] || [ "${KERNEL_FAMILY}" == "bonito" ]; then + # TODO: haven't tested kernel build for coral + if [ "${KERNEL_FAMILY}" == "wahoo" ] || [ "${KERNEL_FAMILY}" == "crosshatch" ] || [ "${KERNEL_FAMILY}" == "bonito" ] || [ "${KERNEL_FAMILY}" == "coral" ]; then export PATH="${BUILD_DIR}/prebuilts/clang/host/linux-x86/clang-r353983c/bin:${PATH}"; export LD_LIBRARY_PATH="${BUILD_DIR}/prebuilts/clang/host/linux-x86/clang-r353983c/lib64:${LD_LIBRARY_PATH}"; make O=out ARCH=arm64 ${KERNEL_DEFCONFIG}_defconfig; @@ -1118,6 +1128,14 @@ release() { --avb_system_key "$KEY_DIR/avb.pem" --avb_system_algorithm SHA256_RSA2048) ;; + vbmeta_chained_v2) + AVB_SWITCHES=(--avb_vbmeta_key "$KEY_DIR/avb.pem" + --avb_vbmeta_algorithm SHA256_RSA2048 + --avb_system_key "$KEY_DIR/avb.pem" + --avb_system_algorithm SHA256_RSA2048 + --avb_vbmeta_system_key "$KEY_DIR/avb.pem" + --avb_vbmeta_system_algorithm SHA256_RSA2048) + ;; esac export PATH=$BUILD_DIR/prebuilts/build-tools/linux-x86/bin:$PATH From 7db78a6a9148989c23dd9244129992c4f75a9969 Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Thu, 18 Jun 2020 21:44:57 -0700 Subject: [PATCH 20/23] fix issue with aosp-branch override not working - closes #162 --- templates/build_template.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/templates/build_template.go b/templates/build_template.go index bc4ee593..89c95733 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -206,10 +206,12 @@ get_latest_versions() { echo "AOSP_VENDOR_BUILD=${AOSP_VENDOR_BUILD}" echo "AOSP_BUILD=${AOSP_BUILD}" - AOSP_BRANCH=$(curl --fail -s "${RATTLESNAKEOS_LATEST_JSON_AOSP}" | jq -r ".${DEVICE}.branch") - if [ -z "AOSP_BUILD" ]; then - aws_notify_simple "ERROR: Unable to get latest AOSP branch details. Stopping build." - exit 1 + if [ -z "$AOSP_BRANCH" ]; then + AOSP_BRANCH=$(curl --fail -s "${RATTLESNAKEOS_LATEST_JSON_AOSP}" | jq -r ".${DEVICE}.branch") + if [ -z "$AOSP_BRANCH" ]; then + aws_notify_simple "ERROR: Unable to get latest AOSP branch details. Stopping build." + exit 1 + fi fi echo "AOSP_BRANCH=${AOSP_BRANCH}" From 3c09d9790c5b0bf44c22aff60851c874af7ab5cc Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Thu, 20 Aug 2020 22:01:08 -0700 Subject: [PATCH 21/23] add pixel 4a support, remove attestation feature --- .gitignore | 1 + README.md | 25 +-- VERSION | 2 +- cli/deploy.go | 69 +++---- stack/aws.go | 47 +++-- templates/build_template.go | 132 +------------ templates/terraform_template.go | 321 -------------------------------- 7 files changed, 57 insertions(+), 540 deletions(-) diff --git a/.gitignore b/.gitignore index 42f4d749..8e3470be 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ rattlesnakeos-stack .vscode/ *.toml .DS_Store +.idea/ diff --git a/README.md b/README.md index 4317b51b..85d510ee 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ RattlesnakeOS is a privacy and security focused Android OS for Google Pixel phon * Legacy Support for Google Pixel, Pixel XL. These devices no longer receive security updates and will eventually be deprecated. * Monthly software and firmware security fixes delivered through built in OTA updater * [Verified boot](https://source.android.com/security/verifiedboot/) with a locked bootloader just like official Android but with your own personal signing keys -* Optional support for [remote attestation](#how-does-the-remote-attestation-feature-work) using [Auditor](https://github.com/GrapheneOS/Auditor) and [AttestationServer](https://github.com/GrapheneOS/AttestationServer) * Latest stable Chromium [browser](https://www.chromium.org) and [webview](https://www.chromium.org/developers/how-tos/build-instructions-android-webview) * Latest stable [F-Droid](https://f-droid.org/) app store and [privileged extension](https://gitlab.com/fdroid/privileged-extension) * Free of Google’s apps and services @@ -34,7 +33,6 @@ Rather than providing random binaries of RattlesnakeOS to install on your phone, * [General](#general) * [Costs](#costs) * [Builds](#builds) - * [Remote Attestation](#remote-attestation) * [Customizations](#customizations) * [Security](#security) * [Uninstalling](#uninstalling) @@ -128,9 +126,8 @@ Or you can specify a different config file to use ``` #### Advanced Examples -Here is an example of a more advanced config file that: enables deployment of a personal attestation server, locks to a specific version of Chromium, specifies a hosts file to install, uses a larger EC2 instance type, builds every 2 days, and pulls in custom patches from the [community patches repo](https://github.com/RattlesnakeOS/community_patches). You can read more about [advanced customization options in FAQ](#customizations). +Here is an example of a more advanced config file that: locks to a specific version of Chromium, specifies a hosts file to install, uses a larger EC2 instance type, builds every 2 days, and pulls in custom patches from the [community patches repo](https://github.com/RattlesnakeOS/community_patches). You can read more about [advanced customization options in FAQ](#customizations). ```toml -attestation-server = true chromium-version = "80.0.3971.4" device = "crosshatch" email = "user@domain.com" @@ -164,9 +161,6 @@ Usage: rattlesnakeos-stack deploy [flags] Flags: - --attestation-instance-type string instance type to use for attestation server. (default "t3.nano") - --attestation-max-price string max ec2 spot instance bid for attestation server. if this value is too low, you may not launch an instance. (default ".005") - --attestation-server deploys and configures a personal attestation server --chromium-version string specify the version of Chromium you want (e.g. 69.0.3497.100) to pin to. if not specified, the latest stable version of Chromium is used. -d, --device string device you want to build for (e.g. crosshatch): to list supported devices use '-d list' -e, --email string email address you want to use for build notifications @@ -251,7 +245,6 @@ No. RattlesnakeOS was created initially as an alternative to [CopperheadOS](http The costs are going to be variable by AWS region and by day and time you are running your builds as spot instances have a variable price depending on market demand. Below is an example scenario that should give you a rough estimate of costs: * The majority of the cost will come from builds on EC2. It currently launches spot instances of type c5.4xlarge which average maybe $.30 an hour in us-west-2 (will vary by region) but can get up over $1 an hour depending on the day and time. You can modify the default `max-price` config value to set the max price you are willing to pay and if market price exceeds that then your instance will be terminated. Builds can take anywhere from 2-6 hours depending on if Chromium needs to be built. So let's say you're doing a build every month at $0.50 an hour and it is taking on average 4 hours - you'd pay ~$2 in EC2 costs per month. * The other very minimal cost would be S3. Storage costs are almost non existent as a stack will only store about 3GB worth of files (factory image, ota file, target file) and at $0.023 per GB you're looking at $0.07 per month in S3 storage costs. The other S3 cost would be for data transfer out for OTA updates - let's say you are just downloading an update per month (~500MB file) at $0.09 per GB you're looking at $0.05 per month in S3 network costs. - * If you are running the optional attestation server, the costs will be around $3-$5/month. This cost comes from an EC2 spot instance and EBS volumes it uses. You can modify the default `attestation-max-price` to a lower value if you want to try to reduce costs further, but if you go to low you may never launch an instance. ### Builds #### How do I change build frequency? @@ -273,22 +266,6 @@ There is a flag you can pass `rattlesnakeos-stack` called `--prevent-shutdown` t #### Why did my EC2 instance randomly terminate? If there wasn't an error notification, this is likely because the [Spot Instance](https://aws.amazon.com/ec2/spot/) max price was not high enough or EC2 is low on capacity and needs to reclaim instances. You can see historical spot instance pricing in the [EC2 console](https://console.aws.amazon.com/ec2sp/v1/spot/home). Click `Pricing History`, select c5.4xlarge for `Instance Type` and pick a date range. I would recommend not setting your `max-price` beyond the on demand price. -### Remote Attestation -#### What is remote attestation? -You can read more about remote attestation [here](https://attestation.app/about). - -#### How does the remote attestation feature work? -There are two primary pieces involved: -* [AttestationServer](https://github.com/GrapheneOS/AttestationServer) - this is the server component used for remote attestation and it is deployed as an [Elastic Beanstalk](https://aws.amazon.com/elasticbeanstalk/) Docker application running on a spot instance to keep costs to a minimum. A number of things are automated here like: SSL certificate generation with Letsencrypt, backups of sqlite database to S3, customization of app to use your personal keys, etc. -* [Auditor](https://github.com/GrapheneOS/Auditor) - this app is built with your keys and updated to point at your personal AttestationServer. - -Steps to setup: -* On your computer, set `attestation-server = true` option in your config and `deploy`. This will setup all of the AWS infrastructure for running AttestationServer, but AttestationServer code isn't deployed yet. Note: you should get a few emails about the app being setup and an instance launching. -* On your computer, start a RattlesnakeOS build with the `build` command. As part of the build process AttestationServer is deployed (should get more emails) and the Auditor app will be built and included into your resulting RattlesnakeOS build. -* Install the resulting OS build on your phone. -* In emails (or within the Auditor app) you should have seen a URL that looks like https://attestation.azmgdmnqbn.us-west-2.elasticbeanstalk.com; visit this in your computer browser, create an account and don't lose the username/password as this has been locked down to only a single user registration. -* On your phone, in the Auditor app click the button in the top right, 'Enable remote verification', and then scan the barcode. - ### Customizations #### How do I customize RattlesnakeOS builds? There are some advanced options that allow you to customize RattlesnakeOS builds to your liking by adding additional patches and prebuilt apps. These can only be setup in the config file and not through CLI flags. diff --git a/VERSION b/VERSION index 82f3d338..6d6c7c85 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -10.0.9 +10.0.10 diff --git a/cli/deploy.go b/cli/deploy.go index 0f629634..c3b6b28a 100644 --- a/cli/deploy.go +++ b/cli/deploy.go @@ -20,8 +20,7 @@ const minimumChromiumVersion = 80 var name, region, email, device, sshKey, maxPrice, skipPrice, schedule string var instanceType, instanceRegions, hostsFile, chromiumVersion string -var attestationMaxPrice, attestationInstanceType string -var preventShutdown, ignoreVersionChecks, encryptedKeys, saveConfig, attestationServer bool +var preventShutdown, ignoreVersionChecks, encryptedKeys, saveConfig bool var patches = &stack.CustomPatches{} var scripts = &stack.CustomScripts{} var prebuilts = &stack.CustomPrebuilts{} @@ -35,11 +34,11 @@ var supportedRegions = []string{"ap-northeast-1", "ap-northeast-2", "ap-northeas var supportedDevicesFriendly = []string{ "Pixel", "Pixel XL", "Pixel 2", "Pixel 2 XL", "Pixel 3", "Pixel 3 XL", "Pixel 3a", "Pixel 3a XL", - "Pixel 4", "Pixel 4 XL"} + "Pixel 4", "Pixel 4 XL", "Pixel 4a"} var supportedDevicesCodename = []string{ "sailfish", "marlin", "walleye", "taimen", "blueline", "crosshatch", "sargo", "bonito", - "flame", "coral"} + "flame", "coral", "sunfish"} var supportDevicesOutput string func init() { @@ -120,17 +119,6 @@ func init() { flags.BoolVar(&preventShutdown, "prevent-shutdown", false, "for debugging purposes only - will prevent ec2 instance from shutting down after build.") - - flags.BoolVar(&attestationServer, "attestation-server", false, "deploys and configures a personal attestation server (Pixel 3/Pixel 3 XL only)") - viper.BindPFlag("attestation-server", flags.Lookup("attestation-server")) - - flags.StringVar(&attestationMaxPrice, "attestation-max-price", ".005", - "max ec2 spot instance price for attestation server. if this value is too low, you may not launch an instance.") - viper.BindPFlag("attestation-max-price", flags.Lookup("attestation-max-price")) - - flags.StringVar(&attestationInstanceType, "attestation-instance-type", "t3.nano", - "instance type to use for attestation server.") - viper.BindPFlag("attestation-instance-type", flags.Lookup("attestation-instance-type")) } var deployCmd = &cobra.Command{ @@ -168,12 +156,6 @@ var deployCmd = &cobra.Command{ return fmt.Errorf("pinned chromium-version must have major version of at least %v", minimumChromiumVersion) } } - if viper.GetBool("attestation-server") { - if viper.GetString("device") != "crosshatch" && viper.GetString("device") != "blueline" && - viper.GetString("device") != "sargo" && viper.GetString("device") != "bonito" { - return errors.New("attestation-server is only supported for pixel 3 devices") - } - } if viper.GetString("force-build") != "" { log.Warnf("The force-build setting has been deprecated and can be removed from your config file. it has been replaced with ignore-version-checks.") } @@ -235,30 +217,27 @@ var deployCmd = &cobra.Command{ } s, err := stack.NewAWSStack(&stack.AWSStackConfig{ - Name: viper.GetString("name"), - Region: viper.GetString("region"), - Device: viper.GetString("device"), - Email: viper.GetString("email"), - InstanceType: viper.GetString("instance-type"), - InstanceRegions: viper.GetString("instance-regions"), - SSHKey: viper.GetString("ssh-key"), - SkipPrice: viper.GetString("skip-price"), - MaxPrice: viper.GetString("max-price"), - Schedule: viper.GetString("schedule"), - ChromiumVersion: viper.GetString("chromium-version"), - HostsFile: viper.GetString("hosts-file"), - EncryptedKeys: viper.GetBool("encrypted-keys"), - IgnoreVersionChecks: viper.GetBool("ignore-version-checks"), - CustomPatches: patches, - CustomScripts: scripts, - CustomPrebuilts: prebuilts, - CustomManifestRemotes: manifestRemotes, - CustomManifestProjects: manifestProjects, - PreventShutdown: preventShutdown, - Version: version, - EnableAttestation: viper.GetBool("attestation-server"), - AttestationInstanceType: viper.GetString("attestation-instance-type"), - AttestationMaxSpotPrice: viper.GetString("attestation-max-price"), + Name: viper.GetString("name"), + Region: viper.GetString("region"), + Device: viper.GetString("device"), + Email: viper.GetString("email"), + InstanceType: viper.GetString("instance-type"), + InstanceRegions: viper.GetString("instance-regions"), + SSHKey: viper.GetString("ssh-key"), + SkipPrice: viper.GetString("skip-price"), + MaxPrice: viper.GetString("max-price"), + Schedule: viper.GetString("schedule"), + ChromiumVersion: viper.GetString("chromium-version"), + HostsFile: viper.GetString("hosts-file"), + EncryptedKeys: viper.GetBool("encrypted-keys"), + IgnoreVersionChecks: viper.GetBool("ignore-version-checks"), + CustomPatches: patches, + CustomScripts: scripts, + CustomPrebuilts: prebuilts, + CustomManifestRemotes: manifestRemotes, + CustomManifestProjects: manifestProjects, + PreventShutdown: preventShutdown, + Version: version, }) if err != nil { log.Fatal(err) diff --git a/stack/aws.go b/stack/aws.go index 29e11843..422ebeeb 100644 --- a/stack/aws.go +++ b/stack/aws.go @@ -49,31 +49,28 @@ type CustomManifestProjects []struct { } type AWSStackConfig struct { - Name string - Region string - Device string - Email string - InstanceType string - InstanceRegions string - SkipPrice string - MaxPrice string - SSHKey string - PreventShutdown bool - Version string - Schedule string - IgnoreVersionChecks bool - ChromiumVersion string - CustomPatches *CustomPatches - CustomScripts *CustomScripts - CustomPrebuilts *CustomPrebuilts - CustomManifestRemotes *CustomManifestRemotes - CustomManifestProjects *CustomManifestProjects - HostsFile string - EncryptedKeys bool - AMI string - EnableAttestation bool - AttestationMaxSpotPrice string - AttestationInstanceType string + Name string + Region string + Device string + Email string + InstanceType string + InstanceRegions string + SkipPrice string + MaxPrice string + SSHKey string + PreventShutdown bool + Version string + Schedule string + IgnoreVersionChecks bool + ChromiumVersion string + CustomPatches *CustomPatches + CustomScripts *CustomScripts + CustomPrebuilts *CustomPrebuilts + CustomManifestRemotes *CustomManifestRemotes + CustomManifestProjects *CustomManifestProjects + HostsFile string + EncryptedKeys bool + AMI string } type AWSStack struct { diff --git a/templates/build_template.go b/templates/build_template.go index 89c95733..c7701005 100644 --- a/templates/build_template.go +++ b/templates/build_template.go @@ -49,6 +49,12 @@ case "$DEVICE" in KERNEL_DEFCONFIG=coral AVB_MODE=vbmeta_chained_v2 ;; + sunfish) + DEVICE_FAMILY=sunfish + KERNEL_FAMILY=sunfish + KERNEL_DEFCONFIG=sunfish + AVB_MODE=vbmeta_chained_v2 + ;; *) echo "warning: unknown device $DEVICE, using Pixel 3 defaults" DEVICE_FAMILY=$1 @@ -110,12 +116,7 @@ BUILD_CHANNEL="stable" # user customizable things HOSTS_FILE=<% .HostsFile %> -# attestion server -ENABLE_ATTESTATION=<% .EnableAttestation %> -ATTESTATION_MAX_SPOT_PRICE=<% .AttestationMaxSpotPrice %> - # aws settings -AWS_ATTESTATION_BUCKET="${STACK_NAME}-attestation" AWS_KEYS_BUCKET="${STACK_NAME}-keys" AWS_ENCRYPTED_KEYS_BUCKET="${STACK_NAME}-keys-encrypted" AWS_RELEASE_BUCKET="${STACK_NAME}-release" @@ -318,9 +319,6 @@ full_run() { aosp_repo_modifications aosp_repo_sync aws_import_keys - if [ "${ENABLE_ATTESTATION}" == "true" ]; then - attestation_setup - fi setup_vendor build_fdroid apply_patches @@ -361,116 +359,6 @@ build_fdroid() { popd } -attestation_setup() { - sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libffi-dev - - cd $HOME - echo "cloning beanstalk cli" - git clone https://github.com/aws/aws-elastic-beanstalk-cli-setup.git - retry ./aws-elastic-beanstalk-cli-setup/scripts/bundled_installer - - PLATFORM_CERT_SHA256=$(openssl x509 -noout -fingerprint -sha256 -inform pem -in ${KEYS_DIR}/${DEVICE}/platform.x509.pem | awk -F"=" '{print $2}' | sed 's/://g') - ATTESTATION_DOMAIN=$(aws --region ${REGION} elasticbeanstalk describe-environments | jq -r '.Environments[] | select(.EnvironmentName=="attestation" and .Status!="Terminated") | .CNAME') - echo "ATTESTATION_DOMAIN: ${ATTESTATION_DOMAIN}" - echo "PLATFORM_CERT_SHA256: ${PLATFORM_CERT_SHA256}" - - OG_PIXEL3_FINGERPRINT="0F9A9CC8ADE73064A54A35C5509E77994E3AA37B6FB889DD53AF82C3C570C5CF" - OG_PIXEL3_XL_FINGERPRINT="06DD526EE9B1CB92AA19D9835B68B4FF1A48A3AD31D813F27C9A7D6C271E9451" - OG_PIXEL3A_FINGERPRINT="3ADD526EE9B1CB92AA19D9835B68B4FF1A48A3AD31D813F27C9A7D6C271E9451" - - PIXEL3_FINGERPRINT=${OG_PIXEL3_FINGERPRINT} - PIXEL3_XL_FINGERPRINT=${OG_PIXEL3_XL_FINGERPRINT} - PIXEL3A_FINGERPRINT=${OG_PIXEL3A_FINGERPRINT} - if [ "${DEVICE}" == "blueline" ]; then - PIXEL3_FINGERPRINT=$(cat ${KEYS_DIR}/${DEVICE}/avb_pkmd.bin | sha256sum | awk '{print $1}' | awk '{ print toupper($0) }') - fi - if [ "${DEVICE}" == "crosshatch" ]; then - PIXEL3_XL_FINGERPRINT=$(cat ${KEYS_DIR}/${DEVICE}/avb_pkmd.bin | sha256sum | awk '{print $1}' | awk '{ print toupper($0) }') - fi - if [ "${DEVICE}" == "sargo" ] || [ "${DEVICE}" == "bonito" ]; then - PIXEL3A_FINGERPRINT=$(cat ${KEYS_DIR}/${DEVICE}/avb_pkmd.bin | sha256sum | awk '{print $1}' | awk '{ print toupper($0) }') - fi - - cd $HOME - echo "cloning and building auditor" - git clone https://github.com/RattlesnakeOS/Auditor.git - cd Auditor - sed -i "s/DOMAIN_NAME/${ATTESTATION_DOMAIN}/g" app/src/main/res/values/strings.xml - sed -i "s/attestation.app/${ATTESTATION_DOMAIN}/" app/src/main/java/app/attestation/auditor/RemoteVerifyJob.java - if [ "${DEVICE}" == "blueline" ]; then - sed -i "s/${OG_PIXEL3_FINGERPRINT}/${PIXEL3_FINGERPRINT}/g" app/src/main/java/app/attestation/auditor/AttestationProtocol.java - fi - if [ "${DEVICE}" == "crosshatch" ]; then - sed -i "s/${OG_PIXEL3_XL_FINGERPRINT}/${PIXEL3_XL_FINGERPRINT}/g" app/src/main/java/app/attestation/auditor/AttestationProtocol.java - fi - if [ "${DEVICE}" == "sargo" ] || [ "${DEVICE}" == "bonito" ]; then - sed -i "s/${OG_PIXEL3A_FINGERPRINT}/${PIXEL3A_FINGERPRINT}/g" app/src/main/java/app/attestation/auditor/AttestationProtocol.java - fi - sed -i "s/990E04F0864B19F14F84E0E432F7A393F297AB105A22C1E1B10B442A4A62C42C/${PLATFORM_CERT_SHA256}/" app/src/main/java/app/attestation/auditor/AttestationProtocol.java - echo "sdk.dir=${HOME}/sdk" > local.properties - echo "sdk.dir=${HOME}/sdk" > app/local.properties - ./gradlew build && ./gradlew assembleRelease - mkdir -p ${BUILD_DIR}/external/Auditor/prebuilt - cp app/build/outputs/apk/release/app-release-unsigned.apk ${BUILD_DIR}/external/Auditor/prebuilt/Auditor.apk - - cd $HOME - echo "cloning attestationserver" - git clone https://github.com/RattlesnakeOS/AttestationServer.git - cd AttestationServer - cat < .ebextensions/.config -option_settings: -- option_name: DOMAIN_NAME - value: ${ATTESTATION_DOMAIN} -- option_name: FINGERPRINT_PIXEL3 - value: ${PIXEL3_FINGERPRINT} -- option_name: FINGERPRINT_PIXEL3_XL - value: ${PIXEL3_XL_FINGERPRINT} -- option_name: FINGERPRINT_PIXEL3A - value: ${PIXEL3A_FINGERPRINT} -- option_name: SNS_ARN - value: ${AWS_SNS_ARN} -- option_name: REGION - value: ${REGION} -- option_name: EC2_SPOT_PRICE - value: ${ATTESTATION_MAX_SPOT_PRICE} -- option_name: S3_BACKUP_BUCKET - value: s3://${AWS_ATTESTATION_BUCKET} -- option_name: ATTESTATION_APP_SIGNATURE_DIGEST_RELEASE - value: ${PLATFORM_CERT_SHA256} -EOF - sed -i "s/STACK_NAME/${STACK_NAME}/g" .ebextensions/01-setup.config - sed -i "s/STACK_NAME/${STACK_NAME}/g" .ebextensions/sqlite-backup-restore.sh - - mkdir -p .elasticbeanstalk - cat < .elasticbeanstalk/config.yml -branch-defaults: - master: - environment: attestation - group_suffix: null -environment-defaults: - attestation: - branch: null - repository: null -global: - application_name: ${AWS_ATTESTATION_BUCKET} - branch: null - default_ec2_keyname: null - default_platform: docker - default_region: ${REGION} - include_git_submodules: true - instance_profile: null - platform_name: null - platform_version: null - profile: null - repository: null - sc: git - workspace_type: Application -EOF - - echo "deploying eb environment" - $HOME/.ebcli-virtual-env/executables/eb deploy attestation -nh -} - get_encryption_key() { additional_message="" if [ "$(aws s3 ls "s3://${AWS_ENCRYPTED_KEYS_BUCKET}/${DEVICE}" | wc -l)" == '0' ]; then @@ -710,9 +598,6 @@ aosp_repo_modifications() { <% end %> <% end %> - <% if .EnableAttestation %> - - <% end %> EOF @@ -907,6 +792,8 @@ patch_device_config() { sed -i 's@PRODUCT_MODEL := AOSP on coral@PRODUCT_MODEL := Pixel 4 XL@' ${BUILD_DIR}/device/google/coral/aosp_coral.mk || true sed -i 's@PRODUCT_MODEL := AOSP on flame@PRODUCT_MODEL := Pixel 4@' ${BUILD_DIR}/device/google/coral/aosp_flame.mk || true + + sed -i 's@PRODUCT_MODEL := AOSP on sunfish@PRODUCT_MODEL := Pixel 4A@' ${BUILD_DIR}/device/google/sunfish/aosp_sunfish.mk || true } get_package_mk_file() { @@ -926,9 +813,6 @@ patch_add_apps() { sed -i "\$aPRODUCT_PACKAGES += F-DroidPrivilegedExtension" ${mk_file} sed -i "\$aPRODUCT_PACKAGES += F-Droid" ${mk_file} sed -i "\$aPRODUCT_PACKAGES += chromium" ${mk_file} - if [ "${ENABLE_ATTESTATION}" == "true" ]; then - sed -i "\$aPRODUCT_PACKAGES += Auditor" ${mk_file} - fi # add any modules defined in custom manifest projects <% if .CustomManifestProjects %><% range $i, $r := .CustomManifestProjects %><% range $j, $q := .Modules %> diff --git a/templates/terraform_template.go b/templates/terraform_template.go index fe38193f..b80c02bf 100644 --- a/templates/terraform_template.go +++ b/templates/terraform_template.go @@ -40,21 +40,6 @@ variable "shell_script_file" { default = "<% .BuildScriptFileLocation %>" } -variable "enable_attestation" { - description = "Whether to enable attestation server" - default = "<% .Config.EnableAttestation %>" -} - -variable "attestation_instance_type" { - description = "Instance type for attestation server" - default = "<% .Config.AttestationInstanceType %>" -} - -variable "attestation_max_spot_price" { - description = "Max spot price for attestation server" - default = "<% .Config.AttestationMaxSpotPrice %>" -} - ################### # Provider ################### @@ -185,54 +170,7 @@ resource "aws_iam_role_policy" "rattlesnake_ec2_policy" { "s3:GetBucketLocation" ], "Resource": "arn:aws:s3:::${var.name}-script" - }<% if .Config.EnableAttestation %>, - { - "Action": [ - "ec2:*", - "autoscaling:*", - "elasticbeanstalk:*" - ], - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "sns:CreateTopic", - "sns:GetTopicAttributes", - "sns:ListSubscriptionsByTopic", - "sns:Subscribe" - ], - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "s3:*" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::elasticbeanstalk-*", - "arn:aws:s3:::elasticbeanstalk-*/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "cloudformation:*" - ], - "Resource": [ - "arn:aws:cloudformation:*:*:stack/awseb-*", - "arn:aws:cloudformation:*:*:stack/eb-*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "iam:PassRole" - ], - "Resource": "arn:aws:iam::*:role/${var.name}-beanstalk-ec2" } - <% end %> ] } EOF @@ -409,265 +347,6 @@ resource "aws_lambda_permission" "allow_cloudwatch_to_call_build_schedule" { source_arn = "${aws_cloudwatch_event_rule.build_schedule.arn}" } -<% if .Config.EnableAttestation %> -################### -# Attestation -################### -data "aws_caller_identity" "current" {} -data "aws_availability_zones" "available" {} - -resource "aws_vpc" "vpc" { - cidr_block = "10.0.0.0/16" - enable_dns_support = "true" - enable_dns_hostnames = "true" - tags { - Name = "${var.name}" - } -} - -resource "aws_subnet" "subnet_public" { - count = "${length(data.aws_availability_zones.available.names)}" - vpc_id = "${aws_vpc.vpc.id}" - cidr_block = "${cidrsubnet("10.0.0.0/16", 8, count.index)}" - availability_zone = "${data.aws_availability_zones.available.names[count.index]}" - tags { - Name = "${var.name}-public-${data.aws_availability_zones.available.names[count.index]}" - } -} - -resource "aws_internet_gateway" "vpc-igw" { - vpc_id = "${aws_vpc.vpc.id}" -} - -resource "aws_route_table" "route_to_igw" { - vpc_id = "${aws_vpc.vpc.id}" - route { - cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.vpc-igw.id}" - } - tags { - Name = "${var.name}-route-to-internet-via-igw" - } -} - -resource "aws_route_table_association" "pub_to_igw_association" { - subnet_id = "${aws_subnet.subnet_public.*.id[count.index]}" - route_table_id = "${aws_route_table.route_to_igw.id}" - count = "${length(data.aws_availability_zones.available.names)}" -} - -resource "aws_security_group" "beanstalk" { - vpc_id = "${aws_vpc.vpc.id}" - name = "${var.name}-beanstalk-attestation-sg" - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - tags { - Name = "${var.name}" - } -} - -resource "aws_s3_bucket" "rattlesnake_s3_attestation" { - bucket = "${var.name}-attestation" - force_destroy = true - acl = "private" - - server_side_encryption_configuration { - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } - } -} - -data "aws_elastic_beanstalk_solution_stack" "docker" { - name_regex = "^64bit Amazon Linux (.*) running Docker(.*)$" - most_recent = true -} - -resource "aws_iam_role" "rattlesnake_beanstalk_ec2_role" { - name = "${var.name}-beanstalk-ec2" - assume_role_policy = < - ################### # Outputs ################### From eddb3c5136317293875fba05937a21d7fa9f818f Mon Sep 17 00:00:00 2001 From: Dan Vittegleo Date: Thu, 27 Aug 2020 21:01:24 -0700 Subject: [PATCH 22/23] add CreateTags and RunInstances to Lambda role - closes #164 --- templates/terraform_template.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/terraform_template.go b/templates/terraform_template.go index b80c02bf..148d1b28 100644 --- a/templates/terraform_template.go +++ b/templates/terraform_template.go @@ -213,6 +213,8 @@ resource "aws_iam_role_policy" "rattlesnake_lambda_policy" { "ec2:DescribeSubnets", "ec2:RequestSpotFleet", "ec2:DescribeSpotPriceHistory", + "ec2:RunInstances", + "ec2:CreateTags", "iam:CreateServiceLinkedRole", "iam:PassRole", "sts:GetCallerIdentity", From 69f873e3e5e664b26faff0a50d2ae9d543a95b95 Mon Sep 17 00:00:00 2001 From: James Anslow Date: Tue, 22 Sep 2020 16:22:22 +0100 Subject: [PATCH 23/23] Updating README.md to reflect changes to awscli output --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 85d510ee..49dd64d5 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Rather than providing random binaries of RattlesnakeOS to install on your phone, ``` keypair_name="rattlesnakeos" ssh-keygen -t rsa -b 4096 -f ${keypair_name} -for region in $(aws ec2 describe-regions --output text | awk '{print $4}'); do +for region in $(aws ec2 describe-regions --output text | awk '{print $3}'); do echo "Importing keypair ${keypair_name} to region ${region}..." aws ec2 import-key-pair --key-name "${keypair_name}" --public-key-material "file://${keypair_name}.pub" --region $region; done