From dd31d9864a9e15b3c1b8e2a4e28835f2708085d2 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 12 May 2025 10:39:13 +0800 Subject: [PATCH] Group log lines in GHA outputs for better readable logs. Signed-off-by: chensuyue --- CodeGen/Dockerfile | 3 +- CodeGen/docker_image_build/build.yaml | 3 ++ CodeGen/tests/test_compose_on_gaudi.sh | 30 ++++++++++-------- CodeGen/tests/test_compose_on_rocm.sh | 35 +++++++++++++-------- CodeGen/tests/test_compose_on_xeon.sh | 27 +++++++++------- CodeGen/tests/test_compose_vllm_on_rocm.sh | 36 ++++++++++++++-------- 6 files changed, 86 insertions(+), 48 deletions(-) diff --git a/CodeGen/Dockerfile b/CodeGen/Dockerfile index 5305a9d89f..74d8bd98f1 100644 --- a/CodeGen/Dockerfile +++ b/CodeGen/Dockerfile @@ -1,8 +1,9 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +ARG IMAGE_REPO=opea ARG BASE_TAG=latest -FROM opea/comps-base:$BASE_TAG +FROM $IMAGE_REPO/comps-base:$BASE_TAG COPY ./codegen.py $HOME/codegen.py diff --git a/CodeGen/docker_image_build/build.yaml b/CodeGen/docker_image_build/build.yaml index 38b28ef80b..282c29766c 100644 --- a/CodeGen/docker_image_build/build.yaml +++ b/CodeGen/docker_image_build/build.yaml @@ -5,6 +5,8 @@ services: codegen: build: args: + IMAGE_REPO: ${REGISTRY} + BASE_TAG: ${TAG} http_proxy: ${http_proxy} https_proxy: ${https_proxy} no_proxy: ${no_proxy} @@ -39,6 +41,7 @@ services: build: context: GenAIComps dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu + extends: codegen image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest} vllm: build: diff --git a/CodeGen/tests/test_compose_on_gaudi.sh b/CodeGen/tests/test_compose_on_gaudi.sh index 58eb0888db..413ce53808 100644 --- a/CodeGen/tests/test_compose_on_gaudi.sh +++ b/CodeGen/tests/test_compose_on_gaudi.sh @@ -27,19 +27,13 @@ export no_proxy=${no_proxy},${ip_address} function build_docker_images() { opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s # Download Gaudi vllm of latest tag git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork @@ -250,24 +244,36 @@ function main() { stop_docker "${docker_compose_profiles[${i}]}" done - # build docker images + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + echo "::endgroup::" # loop all profiles for ((i = 0; i < len_profiles; i++)); do echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}" + + echo "::group::start_services" start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}" + echo "::endgroup::" docker ps -a + echo "::group::validate_microservices" validate_microservices "${docker_llm_container_names[${i}]}" + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + + echo "::group::validate_gradio" validate_gradio + echo "::endgroup::" stop_docker "${docker_compose_profiles[${i}]}" sleep 5s done - echo y | docker system prune + docker system prune -f } main diff --git a/CodeGen/tests/test_compose_on_rocm.sh b/CodeGen/tests/test_compose_on_rocm.sh index d1a6f57309..94f006e358 100644 --- a/CodeGen/tests/test_compose_on_rocm.sh +++ b/CodeGen/tests/test_compose_on_rocm.sh @@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="codegen codegen-ui llm-textgen" @@ -164,18 +158,35 @@ function stop_docker() { function main() { + echo "::group::stop_docker" stop_docker + echo "::endgroup::" + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + echo "::endgroup::" + + echo "::group::start_services" start_services + echo "::endgroup::" + echo "::group::validate_microservices" validate_microservices + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + + echo "::group::validate_frontend" validate_frontend + echo "::endgroup::" + echo "::group::stop_docker" stop_docker - echo y | docker system prune - cd $WORKPATH + echo "::endgroup::" + + docker system prune -f } diff --git a/CodeGen/tests/test_compose_on_xeon.sh b/CodeGen/tests/test_compose_on_xeon.sh index 249f735c8a..4aaa180ec3 100644 --- a/CodeGen/tests/test_compose_on_xeon.sh +++ b/CodeGen/tests/test_compose_on_xeon.sh @@ -27,19 +27,13 @@ export no_proxy=${no_proxy},${ip_address} function build_docker_images() { opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s git clone https://github.com/vllm-project/vllm.git && cd vllm VLLM_VER="v0.8.3" @@ -256,17 +250,28 @@ function main() { for ((i = 0; i < len_profiles; i++)); do echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}" docker ps -a + + echo "::group::start_services" start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}" + echo "::endgroup::" + echo "::group::validate_microservices" validate_microservices "${docker_llm_container_names[${i}]}" + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + + echo "::group::validate_gradio" validate_gradio + echo "::endgroup::" stop_docker "${docker_compose_profiles[${i}]}" sleep 5s done - echo y | docker system prune + docker system prune -f } main diff --git a/CodeGen/tests/test_compose_vllm_on_rocm.sh b/CodeGen/tests/test_compose_vllm_on_rocm.sh index bb75bdafa8..1d78f2a0d7 100644 --- a/CodeGen/tests/test_compose_vllm_on_rocm.sh +++ b/CodeGen/tests/test_compose_vllm_on_rocm.sh @@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="vllm-rocm llm-textgen codegen codegen-ui" @@ -164,17 +158,35 @@ function stop_docker() { function main() { + echo "::group::stop_docker" stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + echo "::endgroup::" + + echo "::group::start_services" start_services + echo "::endgroup::" + echo "::group::validate_microservices" validate_microservices + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + + echo "::group::validate_frontend" validate_frontend + echo "::endgroup::" + echo "::group::stop_docker" stop_docker - echo y | docker system prune - cd $WORKPATH + echo "::endgroup::" + + docker system prune -f }