From 5de1699a6f4ed402faf2a88bfc6b7aa0f6279228 Mon Sep 17 00:00:00 2001 From: oguzkaganozt Date: Fri, 12 Jan 2024 19:28:27 +0300 Subject: [PATCH] Update Docker images and roles --- .../actions/docker-build-and-push/action.yaml | 25 ++++++-- .github/workflows/update-docker-manifest.yaml | 8 +-- ansible/playbooks/openadk.yaml | 16 +++--- ansible/roles/devel/tasks/main.yaml | 8 --- .../roles/geographiclib/defaults/main.yaml | 1 + ansible/roles/geographiclib/tasks/main.yaml | 2 +- docker/README.md | 4 +- docker/autoware-openadk/Dockerfile | 57 +++++++++---------- docker/autoware-openadk/docker-bake.hcl | 17 ++++-- docker/build.sh | 8 ++- docker/run.sh | 7 ++- 11 files changed, 88 insertions(+), 65 deletions(-) diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 330590c70ad..62fb13d337f 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -61,19 +61,31 @@ runs: flavor: | latest=false prefix=${{ inputs.tag-prefix }} - suffix=${{ inputs.tag-suffix }} + suffix=devel${{ inputs.tag-suffix }} + + - name: Docker meta for prebuilt + id: meta-prebuilt + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-prebuilt + flavor: | + latest=false + prefix=${{ inputs.tag-prefix }} + suffix=prebuilt${{ inputs.tag-suffix }} - - name: Docker meta for monorun - id: meta-monorun + - name: Docker meta for runtime + id: meta-runtime uses: docker/metadata-action@v4 with: images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} tags: ${{ steps.set-docker-tags.outputs.tags }} - bake-target: docker-metadata-action-monorun + bake-target: docker-metadata-action-runtime flavor: | latest=false prefix=${{ inputs.tag-prefix }} - suffix=-monorun${{ inputs.tag-suffix }} + suffix=-runtime${{ inputs.tag-suffix }} - name: Login to GitHub Container Registry if: ${{ github.event_name != 'pull_request' }} @@ -91,7 +103,8 @@ runs: files: | docker/${{ inputs.bake-target }}/docker-bake.hcl ${{ steps.meta-devel.outputs.bake-file }} - ${{ steps.meta-monorun.outputs.bake-file }} + ${{ steps.meta-prebuilt.outputs.bake-file }} + ${{ steps.meta-runtime.outputs.bake-file }} provenance: false set: | ${{ inputs.build-args }} diff --git a/.github/workflows/update-docker-manifest.yaml b/.github/workflows/update-docker-manifest.yaml index a0ad71e70e0..d5963ca38e6 100644 --- a/.github/workflows/update-docker-manifest.yaml +++ b/.github/workflows/update-docker-manifest.yaml @@ -28,12 +28,12 @@ jobs: rosdistro: ${{ needs.load-env.outputs.rosdistro }} tag-name: latest - - name: Create alias from 'autoware-openadk:{rosdistro}-latest-monorun' to 'autoware-openadk:latest-monorun' + - name: Create alias from 'autoware-openadk:{rosdistro}-latest-runtime' to 'autoware-openadk:latest-runtime' uses: ./.github/actions/create-main-distro-alias with: package-name: autoware-openadk rosdistro: ${{ needs.load-env.outputs.rosdistro }} - tag-name: latest-monorun + tag-name: latest-runtime - name: Create alias from 'autoware-openadk:{rosdistro}-latest-cuda' to 'autoware-openadk:latest-cuda' uses: ./.github/actions/create-main-distro-alias @@ -42,9 +42,9 @@ jobs: rosdistro: ${{ needs.load-env.outputs.rosdistro }} tag-name: latest-cuda - - name: Create alias from 'autoware-openadk:{rosdistro}-latest-monorun-cuda' to 'autoware-openadk:latest-monorun-cuda' + - name: Create alias from 'autoware-openadk:{rosdistro}-latest-runtime-cuda' to 'autoware-openadk:latest-runtime-cuda' uses: ./.github/actions/create-main-distro-alias with: package-name: autoware-openadk rosdistro: ${{ needs.load-env.outputs.rosdistro }} - tag-name: latest-monorun-cuda + tag-name: latest-runtime-cuda diff --git a/ansible/playbooks/openadk.yaml b/ansible/playbooks/openadk.yaml index ee59a1b61f0..2b53f934d17 100644 --- a/ansible/playbooks/openadk.yaml +++ b/ansible/playbooks/openadk.yaml @@ -5,7 +5,7 @@ - name: Verify OS ansible.builtin.fail: msg: Only Ubuntu 22.04 is supported for this branch. Please refer to https://autowarefoundation.github.io/autoware-documentation/main/installation/autoware/source-installation/. - when: ansible_distribution_version != '22.04' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version != '22.04' - name: Print args ansible.builtin.debug: @@ -34,13 +34,13 @@ when: (module == 'perception-localization' or module == 'all') and prompt_install_nvidia=='y' - role: autoware.dev_env.pacmod when: module == 'planning-control' or module == 'perception-localization' or module == 'all' - - # ONNX files and other artifacts - - role: autoware.dev_env.artifacts - when: module == 'perception-localization' or module == 'all' - + # Development environment - role: autoware.dev_env.devel - when: install_devel == 'true' + when: module == 'dev-tools' - role: autoware.dev_env.ros2_dev_tools - when: install_devel == 'true' + when: module == 'dev-tools' + + # ONNX files and other artifacts + - role: autoware.dev_env.artifacts + when: module == 'dev-tools' or (module == 'all' and install_devel == 'false') diff --git a/ansible/roles/devel/tasks/main.yaml b/ansible/roles/devel/tasks/main.yaml index 0decce0c8fe..e74bc1ea42f 100644 --- a/ansible/roles/devel/tasks/main.yaml +++ b/ansible/roles/devel/tasks/main.yaml @@ -1,11 +1,3 @@ -- name: Install ccache - become: true - ansible.builtin.apt: - name: - - ccache - state: latest - update_cache: true - # https://github.com/git-lfs/git-lfs/wiki/Installation#debian-and-ubuntu - name: Install Git LFS become: true diff --git a/ansible/roles/geographiclib/defaults/main.yaml b/ansible/roles/geographiclib/defaults/main.yaml index e69de29bb2d..897a24acddf 100644 --- a/ansible/roles/geographiclib/defaults/main.yaml +++ b/ansible/roles/geographiclib/defaults/main.yaml @@ -0,0 +1 @@ +module: dev-tools \ No newline at end of file diff --git a/ansible/roles/geographiclib/tasks/main.yaml b/ansible/roles/geographiclib/tasks/main.yaml index 6c474d6406e..9d673cf859f 100644 --- a/ansible/roles/geographiclib/tasks/main.yaml +++ b/ansible/roles/geographiclib/tasks/main.yaml @@ -10,4 +10,4 @@ ansible.builtin.command: cmd: geographiclib-get-geoids egm2008-1 creates: /usr/share/GeographicLib/geoids/egm2008-1.pgm - when: install_devel != 'true' + when: module == 'dev-tools' or install_devel== 'false' \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 6888b894677..ccae2487f26 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,9 +1,9 @@ # Open AD Kit: Containerized Workloads for the Autoware -Open AD Kit offers two types of Docker image to let you get started with the Autoware quickly: `devel` and `monorun`. +Open AD Kit offers two types of Docker image to let you get started with the Autoware quickly: `devel` and `runtime`. 1. The `devel` image enables you to develop Autoware without setting up the local development environment. -2. The `monorun` image contains only runtime executables and enables you to try out Autoware quickly. +2. The `runtime` image contains only runtime executables and enables you to try out Autoware quickly. **Note**: Before proceeding, confirm and agree with the [NVIDIA Deep Learning Container license](https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license). By pulling and using the Autoware Open AD Kit images, you accept the terms and conditions of the license. diff --git a/docker/autoware-openadk/Dockerfile b/docker/autoware-openadk/Dockerfile index 889737189dd..5af80b6fdd3 100644 --- a/docker/autoware-openadk/Dockerfile +++ b/docker/autoware-openadk/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-ins wget \ cmake \ curl \ + ccache \ gnupg \ vim \ unzip \ @@ -38,10 +39,13 @@ RUN --mount=type=ssh \ # Create entrypoint CMD ["/bin/bash"] -FROM base as devel +FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS +ENV CMAKE_GENERATOR=Ninja +ENV CCACHE_DIR=/ccache +ARG ROS_DISTRO # Set up development environment RUN --mount=type=ssh \ @@ -61,8 +65,28 @@ RUN --mount=type=ssh \ && apt update \ && rosdep update \ && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ + && source /opt/ros/"$ROS_DISTRO"/setup.bash \ + && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ + " -Wno-dev" \ + " --no-warn-unused-cli" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + && find /autoware/install -type d -exec chmod 777 {} \; \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ - && rm -rf /autoware + && rm -rf /autoware/build /autoware/src + +CMD ["/bin/bash"] + +FROM prebuilt as devel +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install development tools and artifacts +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y --module dev-tools openadk \ + && pip uninstall -y ansible ansible-core \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Instal gosu for switching to local user in entrypoint RUN --mount=type=ssh \ @@ -78,32 +102,7 @@ RUN chmod +x /dev_entrypoint.sh ENTRYPOINT ["/dev_entrypoint.sh"] CMD ["/bin/bash"] -FROM devel as prebuilt -ENV CMAKE_GENERATOR=Ninja -ENV CCACHE_DIR=/ccache -ARG ROS_DISTRO -WORKDIR /autoware - -# Build and change permission for runtime data conversion -COPY autoware.repos /autoware/ -RUN --mount=type=ssh \ - mkdir src \ - && vcs import src < autoware.repos \ - && source /opt/ros/"$ROS_DISTRO"/setup.bash \ - && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - && find /autoware/install -type d -exec chmod 777 {} \; \ - && echo "source /autoware/install/setup.bash" > /etc/bash.bashrc \ - && rm -rf /autoware/src /autoware/build - -CMD ["/bin/bash"] - -FROM base as monorun +FROM base as runtime SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG LIB_DIR @@ -117,7 +116,7 @@ RUN --mount=type=ssh \ && mkdir src \ && vcs import src < autoware.repos \ && rosdep update \ - && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --dependency-types=exec --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ && find /usr/lib/$LIB_DIR-linux-gnu -name "*.a" -type f -delete \ && find / -name "*.o" -type f -delete \ diff --git a/docker/autoware-openadk/docker-bake.hcl b/docker/autoware-openadk/docker-bake.hcl index d343c04a669..54fef6ef6d4 100644 --- a/docker/autoware-openadk/docker-bake.hcl +++ b/docker/autoware-openadk/docker-bake.hcl @@ -1,10 +1,11 @@ group "default" { - targets = ["devel", "monorun"] + targets = ["prebuilt", "devel", "runtime"] } // For docker/metadata-action target "docker-metadata-action-devel" {} -target "docker-metadata-action-monorun" {} +target "docker-metadata-action-prebuilt" {} +target "docker-metadata-action-runtime" {} target "devel" { inherits = ["docker-metadata-action-devel"] @@ -12,8 +13,14 @@ target "devel" { target = "devel" } -target "monorun" { - inherits = ["docker-metadata-action-monorun"] +target "prebuilt" { + inherits = ["docker-metadata-action-prebuilt"] dockerfile = "docker/autoware-openadk/Dockerfile" - target = "monorun" + target = "prebuilt" +} + +target "runtime" { + inherits = ["docker-metadata-action-runtime"] + dockerfile = "docker/autoware-openadk/Dockerfile" + target = "runtime" } diff --git a/docker/build.sh b/docker/build.sh index a2a870d306b..af3be384b04 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -36,6 +36,11 @@ parse_arguments() { --devel-only) option_devel_only=true ;; + *) + echo "Unknown option: $1" + print_help + exit 1 + ;; esac shift done @@ -115,7 +120,8 @@ build_images() { --set "*.args.SETUP_ARGS=$setup_args" \ --set "*.args.LIB_DIR=$lib_dir" \ --set "devel.tags=ghcr.io/autowarefoundation/autoware-openadk:devel-$rosdistro-latest$image_name_suffix" \ - --set "monorun.tags=ghcr.io/autowarefoundation/autoware-openadk:monorun-$rosdistro-latest$image_name_suffix" \ + --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware-openadk:prebuilt-$rosdistro-latest$image_name_suffix" \ + --set "runtime.tags=ghcr.io/autowarefoundation/autoware-openadk:runtime-$rosdistro-latest$image_name_suffix" \ "${targets[@]}" set +x } diff --git a/docker/run.sh b/docker/run.sh index 94ab431a407..cb8d4dc9a28 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -63,6 +63,11 @@ parse_arguments() { LAUNCH_CMD="$2" shift ;; + *) + echo "Unknown option: $1" + print_help + exit 1 + ;; esac shift done @@ -81,7 +86,7 @@ set_variables() { exit 1 fi MAP="-v ${MAP_PATH}:/${MAP_PATH}" - IMAGE="ghcr.io/autowarefoundation/autoware-openadk:monorun-$rosdistro-latest" + IMAGE="ghcr.io/autowarefoundation/autoware-openadk:runtime-$rosdistro-latest" WORKSPACE="" fi }