From 3f985c77fc490a6bf0734eee57b4a9e03db18b1b Mon Sep 17 00:00:00 2001 From: Niklas Dusenlund Date: Fri, 23 Aug 2024 12:03:19 +0200 Subject: [PATCH 1/2] Dockerfile: Use `python3` executable. Python should be invoked as `python3` since we are writing python3 code and not python2. See https://peps.python.org/pep-0394/#recommendation Many (most?) distributions do not distribute a "python" executable any more, forcing you to pick either 2 or 3. --- .containerversion | 2 +- Dockerfile | 3 --- external/CMakeLists.txt | 1 + scripts/expand_template | 2 +- scripts/get_version | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.containerversion b/.containerversion index 9e5feb525..abac1ea7b 100644 --- a/.containerversion +++ b/.containerversion @@ -1 +1 @@ -46 +47 diff --git a/Dockerfile b/Dockerfile index ca172d1ad..930a8975d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -121,9 +121,6 @@ RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ rm /tmp/protoc-21.2.zip ENV PATH /opt/protoc/bin:$PATH -# Make Python3 the default -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 - # Developer tools RUN apt-get update && apt-get install -y \ bash-completion diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 633307dc8..73cf905cb 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -52,6 +52,7 @@ ExternalProject_Add(libwally-core COMMAND ${CMAKE_COMMAND} -E env "CFLAGS=${LIBWALLY_CFLAGS}" "LDFLAGS=${LIBWALLY_LDFLAGS}" + PYTHON_VERSION=3 ${CMAKE_CURRENT_SOURCE_DIR}/libwally-core/configure ${CONFIGURE_FLAGS} ${LIBWALLY_CONFIGURE_FLAGS} diff --git a/scripts/expand_template b/scripts/expand_template index ec17524a7..e8fb133b0 100755 --- a/scripts/expand_template +++ b/scripts/expand_template @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Expand python templates""" from string import Template diff --git a/scripts/get_version b/scripts/get_version index fe8b88975..a407eb0e6 100755 --- a/scripts/get_version +++ b/scripts/get_version @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Parse the tags and strip the prefix First argument should be prefix to match against From d51692f6ebd5c7f78a0b682a4a14ad0c9a3099af Mon Sep 17 00:00:00 2001 From: Niklas Dusenlund Date: Fri, 23 Aug 2024 12:04:25 +0200 Subject: [PATCH 2/2] Dockerfile: Maintenance All temporary files need to be deleted in every `RUN` command to avoid adding it in a layer. Removing them in a later `RUN` command will still make them bloat the final image. Instead of copying files temporarily they should be mounted in the `RUN` command where they are needed. This avoids the need to delete them at the end of the command. Environment variables that are intended only for when the container is built should be prepended on the lines where they are needed to avoid polluting the environment in the running container. We should strive to have as few layers as possible, but still enough to simplify debugging the container image. When you only have 1 layer it is not possible to create intermediate images and inspect the state of the image. --- .containerversion | 2 +- Dockerfile | 108 ++++++++++++++++++++++------------------------ 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/.containerversion b/.containerversion index abac1ea7b..21e72e8ac 100644 --- a/.containerversion +++ b/.containerversion @@ -1 +1 @@ -47 +48 diff --git a/Dockerfile b/Dockerfile index 930a8975d..639b92d38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,20 +19,23 @@ # $ docker run --privileged --rm tonistiigi/binfmt --install arm64 FROM ubuntu:22.04 -ENV DEBIAN_FRONTEND noninteractive # These are automatically provided by docker (no need for --build-arg) ARG TARGETPLATFORM ARG TARGETARCH -RUN apt-get update && apt-get upgrade -y && apt-get install -y wget nano rsync curl gnupg2 jq unzip bzip2 xz-utils +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y wget nano rsync curl gnupg2 jq unzip bzip2 xz-utils && \ + rm -rf /var/lib/apt/lists/* + -# for clang-*-15, see https://apt.llvm.org/ RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \ echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + rm /root/.wget-hsts -# Install gcc8-arm-none-eabi RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ GNU_TOOLCHAIN=https://developer.arm.com/-/media/Files/downloads/gnu/13.3.rel1/binrel/arm-gnu-toolchain-13.3.rel1-aarch64-arm-none-eabi.tar.xz \ GNU_TOOLCHAIN_HASH=c8824bffd057afce2259f7618254e840715f33523a3d4e4294f471208f976764 \ @@ -42,13 +45,13 @@ RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ GNU_TOOLCHAIN_HASH=fb31fbdfe08406ece43eef5df623c0b2deb8b53e405e2c878300f7a1f303ee52 \ GNU_TOOLCHAIN_FORMAT=bz2; \ fi; \ - wget -O gcc.tar.${GNU_TOOLCHAIN_FORMAT} ${GNU_TOOLCHAIN} &&\ - echo "$GNU_TOOLCHAIN_HASH gcc.tar.${GNU_TOOLCHAIN_FORMAT}" | sha256sum -c &&\ - tar -xvf gcc.tar.${GNU_TOOLCHAIN_FORMAT} -C /usr/local --strip-components=1 &&\ - rm -f gcc.tar.${GNU_TOOLCHAIN_FORMAT} + wget -O gcc.tar.${GNU_TOOLCHAIN_FORMAT} ${GNU_TOOLCHAIN} && \ + echo "$GNU_TOOLCHAIN_HASH gcc.tar.${GNU_TOOLCHAIN_FORMAT}" | sha256sum -c && \ + tar -xvf gcc.tar.${GNU_TOOLCHAIN_FORMAT} -C /usr/local --strip-components=1 && \ + rm -f gcc.tar.${GNU_TOOLCHAIN_FORMAT} /root/.wget-hsts -# Tools for building -RUN apt-get update && apt-get install -y \ +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y \ make \ llvm-18 \ gcc-10 \ @@ -80,36 +83,38 @@ RUN update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100 # Tools for CI RUN apt-get update && apt-get install -y \ + libhidapi-dev \ python3 \ python3-pip \ + doxygen \ + graphviz \ clang-format-18 \ - clang-tidy-18 + clang-tidy-18 \ + bash-completion \ + && rm -rf /var/lib/apt/lists/* -RUN python3 -m pip install --upgrade pip +# Set gcc-10 as the default gcc +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && \ + update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100 && \ + rm /var/log/alternatives.log # Python modules -COPY py/bitbox02 /tmp/bitbox02 -RUN python3 -m pip install /tmp/bitbox02 -RUN rm -r /tmp/bitbox02 -COPY py/requirements.txt /tmp -RUN python3 -m pip install --upgrade --requirement /tmp/requirements.txt -RUN rm /tmp/requirements.txt - -# Python modules for CI -RUN python3 -m pip install --upgrade \ +RUN --mount=source=py,target=/mnt,rw \ + python3 -m pip install --no-compile --no-cache-dir /mnt/bitbox02 && \ + python3 -m pip install --no-compile --no-cache-dir --upgrade --requirement /mnt/requirements.txt && \ + python3 -m pip install --no-compile --no-cache-dir --upgrade \ pylint==2.13.9 \ pylint-protobuf==0.20.2 \ black==22.3.0 \ mypy==0.960 \ - mypy-protobuf==3.2.0 - -# Python modules for packaging -RUN python3 -m pip install --upgrade \ + mypy-protobuf==3.2.0 \ setuptools==41.2.0 \ wheel==0.33.6 \ - twine==1.15.0 + twine==1.15.0 \ + gcovr==7.2 #Install protoc from release, because the version available on the repo is too old +ENV PATH /opt/protoc/bin:$PATH RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ PROTOC_URL=https://github.com/protocolbuffers/protobuf/releases/download/v21.2/protoc-21.2-linux-aarch_64.zip; \ else \ @@ -119,47 +124,36 @@ RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ curl -L0 ${PROTOC_URL} -o /tmp/protoc-21.2.zip && \ unzip /tmp/protoc-21.2.zip -d /opt/protoc && \ rm /tmp/protoc-21.2.zip -ENV PATH /opt/protoc/bin:$PATH - -# Developer tools -RUN apt-get update && apt-get install -y \ - bash-completion -# Install gcovr from PIP to get a newer version than in apt repositories -RUN python3 -m pip install gcovr # Install Go, used for the tools in tools/go and for test/gounittest -ENV GOPATH /opt/go -ENV GOROOT /opt/go_dist/go -ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH +ENV PATH=/opt/go_dist/go/bin:/opt/go/bin:$PATH GOPATH=/opt/go GOROOT=/opt/go_dist/go RUN mkdir -p /opt/go_dist && \ curl https://dl.google.com/go/go1.19.3.linux-${TARGETARCH}.tar.gz | tar -xz -C /opt/go_dist # Install lcov from release (the one from the repos is too old). -RUN cd /opt && wget https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz && tar -xf lcov-1.14.tar.gz -ENV PATH /opt/lcov-1.14/bin:$PATH +ENV PATH=/opt/lcov-1.14/bin:$PATH +RUN curl -L https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz | tar -xz -C /opt # Install rust compiler -ENV PATH /opt/cargo/bin:$PATH -ENV RUSTUP_HOME=/opt/rustup -COPY src/rust/rust-toolchain.toml /tmp/rust-toolchain.toml -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /tmp/rust-toolchain.toml) -y -RUN rustup target add thumbv7em-none-eabi -RUN rustup component add rustfmt -RUN rustup component add clippy -RUN rustup component add rust-src -RUN CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.28.0 --locked -RUN CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.71.1 --locked +# Since bindgen embeds information about its target directory, use a deterministic path for it. +ENV PATH=/opt/cargo/bin:$PATH RUSTUP_HOME=/opt/rustup +RUN --mount=source=tools/prost-build-proto,target=/mnt/prost-build-proto,rw \ + --mount=source=src/rust/rust-toolchain.toml,target=/mnt/rust-toolchain.toml \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /mnt/rust-toolchain.toml) -y && \ + rustup target add thumbv7em-none-eabi && \ + rustup component add rustfmt && \ + rustup component add clippy && \ + rustup component add rust-src && \ + CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.28.0 --locked && \ + CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.71.1 --locked --target-dir=/tmp/bindgen-target && \ + CARGO_HOME=/opt/cargo cargo install --path /mnt/prost-build-proto --locked && \ + rm -r /tmp/bindgen-target /opt/cargo/registry/index /opt/cargo/.global-cache # Until cargo vendor supports vendoring dependencies of the rust std libs we # need a copy of this file next to the toml file. It also has to be world # writable so that invocations of `cargo vendor` can update it. Below is the # tracking issue for `cargo vendor` to support rust std libs. # https://github.com/rust-lang/wg-cargo-std-aware/issues/23 -RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/" -RUN chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock - -COPY tools/prost-build-proto prost-build-proto -RUN CARGO_HOME=/opt/cargo cargo install --path prost-build-proto --locked - -# Clean temporary files to reduce image size -RUN rm -rf /var/lib/apt/lists/* +RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/" && \ + chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock