Merge pull request #3331 from autowarefoundation/release/main-to-awsi…

…m-stable

chore(release) update branch and use awsim-stable for autoware.universe and autoware_launch

.fdignore

@@ -0,0 +1 @@
+!/src/

.github/workflows/docker-build-and-push-main-self-hosted.yaml

@@ -54,6 +54,7 @@ jobs:
             *.platform=linux/arm64
             *.args.ROS_DISTRO=${{ env.rosdistro }}
             *.args.BASE_IMAGE=${{ env[format('{0}', matrix.base_image_env)] }}
+            *.args.PREBUILT_BASE_IMAGE=${{ env.prebuilt_base_image }}
             *.args.SETUP_ARGS=${{ matrix.setup-args }}
           tag-prefix: ${{ env.rosdistro }}-
           tag-suffix: ${{ matrix.additional-tag-suffix }}-arm64

.github/workflows/docker-build-and-push-main.yaml

@@ -49,6 +49,7 @@ jobs:
             *.platform=linux/amd64
             *.args.ROS_DISTRO=${{ env.rosdistro }}
             *.args.BASE_IMAGE=${{ env[format('{0}', matrix.base_image_env)] }}
+            *.args.PREBUILT_BASE_IMAGE=${{ env.prebuilt_base_image }}
             *.args.SETUP_ARGS=${{ matrix.setup-args }}
           tag-prefix: ${{ env.rosdistro }}-
           tag-suffix: ${{ matrix.additional-tag-suffix }}-amd64

.github/workflows/pre-commit-autoupdate.yaml

@@ -7,7 +7,6 @@ on:
 
 jobs:
   check-secret:
-    if: ${{ github.event.repository.private }}
     uses: autowarefoundation/autoware-github-actions/.github/workflows/check-secret.yaml@v1
     secrets:
       secret: ${{ secrets.APP_ID }}

.github/workflows/pre-commit.yaml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   pre-commit:
-    if: ${{ github.event.repository.private }}
+    if: ${{ github.event.repository.private }} # Use pre-commit.ci for public repositories
     runs-on: ubuntu-latest
     steps:
       - name: Generate token

.pre-commit-config.yaml

@@ -1,6 +1,5 @@
 ci:
   autofix_commit_msg: "style(pre-commit): autofix"
-  autoupdate_commit_msg: "ci(pre-commit): autoupdate"
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -18,7 +17,7 @@ repos:
         args: [--markdown-linebreak-ext=md]
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.32.2
+    rev: v0.33.0
     hooks:
       - id: markdownlint
         args: [-c, .markdownlint.yaml, --fix]
@@ -29,23 +28,23 @@ repos:
       - id: prettier
 
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.28.0
+    rev: v1.29.0
     hooks:
       - id: yamllint
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.8.0.4
+    rev: v0.9.0.2
     hooks:
       - id: shellcheck
 
   - repo: https://github.com/scop/pre-commit-shfmt
-    rev: v3.5.1-1
+    rev: v3.6.0-1
     hooks:
       - id: shfmt
         args: [-w, -s, -i=4]
 
   - repo: https://github.com/AleksaC/hadolint-py
-    rev: v2.10.0
+    rev: v2.12.0.2
     hooks:
       - id: hadolint
 

amd64.env

@@ -2,6 +2,7 @@ rosdistro=humble
 rmw_implementation=rmw_cyclonedds_cpp
 base_image=ubuntu:22.04
 cuda_base_image=ubuntu:22.04
+prebuilt_base_image=ubuntu:22.04
 cuda_version=11.6
 cudnn_version=8.4.1.50-1+cuda11.6
 tensorrt_version=8.4.2-1+cuda11.6

ansible/playbooks/universe.yaml

@@ -31,8 +31,10 @@
     # Core
     - role: autoware.dev_env.autoware_core
     - role: autoware.dev_env.ccache
+      when: install_devel == 'true'
     - role: autoware.dev_env.plotjuggler
     - role: autoware.dev_env.pre_commit
+      when: install_devel == 'true'
     - role: autoware.dev_env.ros2
     - role: autoware.dev_env.ros2_dev_tools
     - role: autoware.dev_env.rmw_implementation

autoware.repos

@@ -3,48 +3,48 @@ repositories:
   core/autoware_msgs:
     type: git
     url: https://github.com/autowarefoundation/autoware_msgs.git
-    version: 4f13d4b8b465ed7f424fce9af17882dbe1752875
+    version: 90ed3f1f3371d75eb30b6e7d2acb88ccec4e8030
   core/autoware_adapi_msgs:
     type: git
     url: https://github.com/autowarefoundation/autoware_adapi_msgs.git
     version: 9679b5a7a1f4cfff2fa50b80d2759d3937f2f953
   core/autoware_common:
     type: git
     url: https://github.com/autowarefoundation/autoware_common.git
-    version: 6916df26fafe6749db4b1d5bd6636a92444fc48d
+    version: 34338c0d9e68ea8c7598a66f319cdcb6d92873a3
   core/autoware.core:
     type: git
     url: https://github.com/autowarefoundation/autoware.core.git
-    version: 6bafedfb24fb34157ed65bfe3f6f4c1ed0fbc80b
+    version: 0a3421239fcf576d3e73036c6b5da1daf2b8c8d0
   core/external/autoware_auto_msgs: # TODO(mfc): Remove when autoware_msgs is merged
     type: git
     url: https://github.com/tier4/autoware_auto_msgs.git
-    version: 6b5bc4365f9a2fc913bc11afa74ec21ffa2dbf32
+    version: f6642370c6f4f42a5dc0b6c1fc4a21396d4dc34c
   # universe
   universe/autoware.universe:
     type: git
     url: https://github.com/autowarefoundation/autoware.universe.git
-    version: 9085f2f31f0d8cbba6cbe3ffc3b8ca447631ad32
+    version: awsim-stable
   universe/external/tier4_ad_api_adaptor: # TODO(TIER IV): Improve design/code and transfer to AWF
     type: git
     url: https://github.com/tier4/tier4_ad_api_adaptor.git
-    version: 5084f9c8eaf03458a216060798da2b1e4fa96f28
+    version: e490986fd3a7e35aa5af8f0120d083afdd8a4fce
   universe/external/tier4_autoware_msgs:
     type: git
     url: https://github.com/tier4/tier4_autoware_msgs.git
-    version: a360ee9f5235a0d426427813f26e43027e32139d
+    version: 87febc35abc9310eaec0b6dd3ce2a4845ce515aa
   universe/external/morai_msgs:
     type: git
     url: https://github.com/MORAI-Autonomous/MORAI-ROS2_morai_msgs.git
-    version: 6fd6a711e4bbf8a9989b54028e8074acabbbce6f
+    version: 04f0a0b6a069fef62e0236189ce23d60abfe97f7
   universe/external/muSSP:
     type: git
     url: https://github.com/tier4/muSSP.git
     version: c79e98fd5e658f4f90c06d93472faa977bc873b9
   universe/external/ndt_omp:
     type: git
     url: https://github.com/tier4/ndt_omp.git
-    version: f59e1667390fe66d72c5c3aa0b25385b5b6dd8cf
+    version: eab36ace79c0d5b12b4eba5b900ea6ce665e654f
   universe/external/pointcloud_to_laserscan:
     type: git
     url: https://github.com/tier4/pointcloud_to_laserscan.git
@@ -53,7 +53,7 @@ repositories:
   launcher/autoware_launch:
     type: git
     url: https://github.com/autowarefoundation/autoware_launch.git
-    version: e4abe673667a8d4f2d783ed22edacbf5d4784b8f
+    version: awsim-stable
   # sensor_component
   sensor_component/external/sensor_component_description:
     type: git
@@ -62,20 +62,20 @@ repositories:
   sensor_component/external/tamagawa_imu_driver:
     type: git
     url: https://github.com/tier4/tamagawa_imu_driver.git
-    version: 28ad3cd4fb043e5f92353a540c3531cd4cb7bef3
+    version: de4bf6be79aa2968cf2f62e0ebe1ff8a5797e6ad
   sensor_component/external/velodyne_vls:
     type: git
     url: https://github.com/tier4/velodyne_vls.git
-    version: baeafaf9a376c5798f7b67a77211890c33900f84
+    version: 55dffdf3b3ef889e149977dc6ab1da124b5176c9
   # sensor_kit
   sensor_kit/sample_sensor_kit_launch:
     type: git
     url: https://github.com/autowarefoundation/sample_sensor_kit_launch.git
-    version: 03decbd31bb954eb9f52daaf3a3fa2b921dbb0c3
+    version: 06330e6b07449a3609ced295c0a9ce6a1805ef99
   sensor_kit/external/awsim_sensor_kit_launch: # TODO: Integrate into sample_sensor_kit_launch
     type: git
     url: https://github.com/RobotecAI/awsim_sensor_kit_launch.git
-    version: d9022ee9bbfd958c239b673cfbb230eea50607be
+    version: a1f5993407ffeb4abcf97a49cd1b4034768d97b4
   # vehicle
   vehicle/sample_vehicle_launch:
     type: git
@@ -84,9 +84,9 @@ repositories:
   vehicle/external/pacmod_interface:
     type: git
     url: https://github.com/tier4/pacmod_interface.git
-    version: b5ae20345f2551da0c6e4140a3dc3479d64efd1f
+    version: 3db50a2abc68576308289cc84d83d14376144d64
   # param
   param/autoware_individual_params:
     type: git
     url: https://github.com/autowarefoundation/autoware_individual_params.git
-    version: 79cff0ba014808050be6f5cb3b4764ba2c96c21c
+    version: 09082b6cbb99a840c53878a190616d35a40b569f

docker/autoware-universe/Dockerfile

@@ -1,4 +1,6 @@
+# Image args should come at the beginning.
 ARG BASE_IMAGE
+ARG PREBUILT_BASE_IMAGE
 # hadolint ignore=DL3006
 FROM $BASE_IMAGE as devel
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -43,10 +45,10 @@ RUN rm -rf \
   /etc/apt/sources.list.d/nvidia-docker.list
 
 ## Register Vulkan GPU vendors
-ADD "https://gitlab.com/nvidia/container-images/vulkan/raw/dc389b0445c788901fda1d85be96fd1cb9410164/nvidia_icd.json" /etc/vulkan/icd.d/nvidia_icd.json
-RUN chmod 644 /etc/vulkan/icd.d/nvidia_icd.json
-ADD "https://gitlab.com/nvidia/container-images/opengl/raw/5191cf205d3e4bb1150091f9464499b076104354/glvnd/runtime/10_nvidia.json" /etc/glvnd/egl_vendor.d/10_nvidia.json
-RUN chmod 644 /etc/glvnd/egl_vendor.d/10_nvidia.json
+RUN curl https://gitlab.com/nvidia/container-images/vulkan/raw/dc389b0445c788901fda1d85be96fd1cb9410164/nvidia_icd.json -o /etc/vulkan/icd.d/nvidia_icd.json \
+  && chmod 644 /etc/vulkan/icd.d/nvidia_icd.json
+RUN curl https://gitlab.com/nvidia/container-images/opengl/raw/5191cf205d3e4bb1150091f9464499b076104354/glvnd/runtime/10_nvidia.json -o /etc/glvnd/egl_vendor.d/10_nvidia.json \
+  && chmod 644 /etc/glvnd/egl_vendor.d/10_nvidia.json
 
 ## Register OpenCL GPU vendors
 RUN mkdir -p /etc/OpenCL/vendors \
@@ -58,14 +60,69 @@ RUN mkdir -p /etc/OpenCL/vendors \
 RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" > /etc/bash.bashrc
 CMD ["/bin/bash"]
 
-FROM devel as prebuilt
+FROM devel as builder
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ## Build and change permission for runtime data conversion
 RUN source /opt/ros/"$ROS_DISTRO"/setup.bash \
   && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release \
   && find /autoware/install -type d -exec chmod 777 {} \;
 
+# hadolint ignore=DL3006
+FROM $PREBUILT_BASE_IMAGE as prebuilt
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ARG ROS_DISTRO
+ARG SETUP_ARGS
+
+## Install apt packages
+# hadolint ignore=DL3008
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
+  git \
+  ssh \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+## Copy files
+COPY autoware.repos setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/
+COPY ansible/ /autoware/ansible/
+WORKDIR /autoware
+RUN ls /autoware
+
+## Add GitHub to known hosts for private repositories
+RUN mkdir -p ~/.ssh \
+  && ssh-keyscan github.com >> ~/.ssh/known_hosts
+
+## Set up runtime environment
+RUN --mount=type=ssh \
+  ./setup-dev-env.sh -y $SETUP_ARGS --no-cuda-drivers --runtime universe \
+  && pip uninstall -y ansible ansible-core \
+  && mkdir src \
+  && vcs import src < autoware.repos \
+  && rosdep update \
+  && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \
+  && rm -rf src \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+## Copy install folder from builder
+COPY --from=builder /autoware/install/ /autoware/install/
+
+## Clean up unnecessary files
+RUN rm -rf \
+  "$HOME"/.cache \
+  /etc/apt/sources.list.d/cuda*.list \
+  /etc/apt/sources.list.d/docker.list \
+  /etc/apt/sources.list.d/nvidia-docker.list
+
+## Register Vulkan GPU vendors
+ADD "https://gitlab.com/nvidia/container-images/vulkan/raw/dc389b0445c788901fda1d85be96fd1cb9410164/nvidia_icd.json" /etc/vulkan/icd.d/nvidia_icd.json
+RUN chmod 644 /etc/vulkan/icd.d/nvidia_icd.json
+ADD "https://gitlab.com/nvidia/container-images/opengl/raw/5191cf205d3e4bb1150091f9464499b076104354/glvnd/runtime/10_nvidia.json" /etc/glvnd/egl_vendor.d/10_nvidia.json
+RUN chmod 644 /etc/glvnd/egl_vendor.d/10_nvidia.json
+
 ## Create entrypoint
+# hadolint ignore=DL3059
 RUN echo "source /autoware/install/setup.bash" > /etc/bash.bashrc
 CMD ["/bin/bash"]

docker/build.sh

@@ -16,6 +16,9 @@ while [ "$1" != "" ]; do
         option_platform="$2"
         shift
         ;;
+    --no-prebuilt)
+        option_no_prebuilt=true
+        ;;
     *)
         args+=("$1")
         ;;
@@ -32,6 +35,14 @@ else
     image_name_suffix="-cuda"
 fi
 
+# Set prebuilt options
+if [ "$option_no_prebuilt" = "true" ]; then
+    targets=("devel")
+else
+    # default targets include devel and prebuilt
+    targets=()
+fi
+
 # Set platform
 if [ -n "$option_platform" ]; then
     platform="$option_platform"
@@ -52,13 +63,15 @@ fi
 export BUILDKIT_STEP_LOG_MAX_SIZE=10000000
 
 set -x
-docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/autoware-universe/docker-bake.hcl" \
+docker buildx bake --no-cache --load --progress=plain -f "$SCRIPT_DIR/autoware-universe/docker-bake.hcl" \
     --set "*.context=$WORKSPACE_ROOT" \
     --set "*.ssh=default" \
     --set "*.platform=$platform" \
     --set "*.args.ROS_DISTRO=$rosdistro" \
     --set "*.args.BASE_IMAGE=$base_image" \
+    --set "*.args.PREBUILT_BASE_IMAGE=$prebuilt_base_image" \
     --set "*.args.SETUP_ARGS=$setup_args" \
     --set "devel.tags=ghcr.io/autowarefoundation/autoware-universe:$rosdistro-latest$image_name_suffix" \
-    --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware-universe:$rosdistro-latest-prebuilt$image_name_suffix"
+    --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware-universe:$rosdistro-latest-prebuilt$image_name_suffix" \
+    "${targets[@]}"
 set +x

setup-dev-env.sh

@@ -27,9 +27,9 @@ while [ "$1" != "" ]; do
         # Disable installation of 'cuda-drivers' in the role 'cuda'.
         option_no_cuda_drivers=true
         ;;
-    --no-dev)
-        # Disable installation dev packages .
-        option_no_dev=true
+    --runtime)
+        # Disable installation dev package of role 'cuda' and 'tensorrt'.
+        option_runtime=true
         ;;
     *)
         args+=("$1")
@@ -82,8 +82,9 @@ if [ "$option_no_cuda_drivers" = "true" ]; then
 fi
 
 # Check installation of dev package
-if [ "$option_no_dev" = "true" ]; then
+if [ "$option_runtime" = "true" ]; then
     ansible_args+=("--extra-vars" "install_devel=false")
+    # ROS installation type, default "desktop"
     ansible_args+=("--extra-vars" "installation_type=ros-base")
 else
     ansible_args+=("--extra-vars" "install_devel=true")
@@ -113,10 +114,15 @@ if ! (command -v git >/dev/null 2>&1); then
     sudo apt-get -y install git
 fi
 
+# Install pip for ansible
+if ! (python3 -m pip --version >/dev/null 2>&1); then
+    sudo apt-get -y update
+    sudo apt-get -y install python3-pip python3-venv
+fi
+
 # Install pipx for ansible
 if ! (python3 -m pipx --version >/dev/null 2>&1); then
     sudo apt-get -y update
-    sudo apt-get -y install python3-pip python3-venv
     python3 -m pip install --user pipx
 fi
 

simulator.repos

@@ -2,4 +2,4 @@ repositories:
   simulator/scenario_simulator:
     type: git
     url: https://github.com/tier4/scenario_simulator_v2.git
-    version: eb5e3220b31ef26b49939f748e82b46f28ce80e0
+    version: c425a60edc321fcfd56d4404ec42f0a64865d3e4