From f2d426300b221ba633789e07a49a8a454c54b71e Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 12 May 2025 11:25:29 +0800 Subject: [PATCH] Group log lines in GHA outputs for better readable logs. Signed-off-by: chensuyue --- DocIndexRetriever/Dockerfile | 3 +- .../docker_image_build/build.yaml | 2 ++ .../tests/test_compose_milvus_on_gaudi.sh | 34 +++++++++++-------- .../tests/test_compose_milvus_on_xeon.sh | 33 +++++++++++------- .../tests/test_compose_on_gaudi.sh | 29 ++++++++++------ .../tests/test_compose_on_xeon.sh | 30 ++++++++++------ .../test_compose_without_rerank_on_xeon.sh | 27 +++++++++++---- 7 files changed, 102 insertions(+), 56 deletions(-) diff --git a/DocIndexRetriever/Dockerfile b/DocIndexRetriever/Dockerfile index 06fb1dc016..a9aa823f5c 100644 --- a/DocIndexRetriever/Dockerfile +++ b/DocIndexRetriever/Dockerfile @@ -1,8 +1,9 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +ARG IMAGE_REPO=opea ARG BASE_TAG=latest -FROM opea/comps-base:$BASE_TAG +FROM $IMAGE_REPO/comps-base:$BASE_TAG COPY ./retrieval_tool.py $HOME/retrieval_tool.py diff --git a/DocIndexRetriever/docker_image_build/build.yaml b/DocIndexRetriever/docker_image_build/build.yaml index 80753e8946..bfaaf51d34 100644 --- a/DocIndexRetriever/docker_image_build/build.yaml +++ b/DocIndexRetriever/docker_image_build/build.yaml @@ -5,6 +5,8 @@ services: doc-index-retriever: build: args: + IMAGE_REPO: ${REGISTRY:-opea} + BASE_TAG: ${TAG:-latest} http_proxy: ${http_proxy} https_proxy: ${https_proxy} no_proxy: ${no_proxy} diff --git a/DocIndexRetriever/tests/test_compose_milvus_on_gaudi.sh b/DocIndexRetriever/tests/test_compose_milvus_on_gaudi.sh index 40633be8f4..837ed8a0d2 100644 --- a/DocIndexRetriever/tests/test_compose_milvus_on_gaudi.sh +++ b/DocIndexRetriever/tests/test_compose_milvus_on_gaudi.sh @@ -20,17 +20,15 @@ function build_docker_images() { if [ ! -d "GenAIComps" ] ; then git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git fi + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s + service_list="dataprep embedding retriever reranking doc-index-retriever" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 - docker pull ghcr.io/huggingface/tei-gaudi:1.5.0 - docker pull quay.io/coreos/etcd:v3.5.5 - docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z - docker pull milvusdb/milvus:v2.4.6 docker images && sleep 1s - - echo "Docker images built!" } function start_services() { @@ -127,19 +125,27 @@ function stop_docker() { function main() { + echo "::group::stop_docker" stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi - echo "Dump current docker ps" - docker ps - start_time=$(date +%s) + echo "::endgroup::" + + echo "::group::start_services" start_services - end_time=$(date +%s) - duration=$((end_time-start_time)) - echo "Mega service start duration is $duration s" + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + echo "::group::stop_docker" stop_docker - echo y | docker system prune + echo "::endgroup::" + + docker system prune -f } diff --git a/DocIndexRetriever/tests/test_compose_milvus_on_xeon.sh b/DocIndexRetriever/tests/test_compose_milvus_on_xeon.sh index 59b1c40aa0..b4ff57a89a 100755 --- a/DocIndexRetriever/tests/test_compose_milvus_on_xeon.sh +++ b/DocIndexRetriever/tests/test_compose_milvus_on_xeon.sh @@ -20,16 +20,15 @@ function build_docker_images() { if [ ! -d "GenAIComps" ] ; then git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git fi + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s + service_list="dataprep embedding retriever reranking doc-index-retriever" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 - docker pull quay.io/coreos/etcd:v3.5.5 - docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z - docker pull milvusdb/milvus:v2.4.6 docker images && sleep 1s - - echo "Docker images built!" } function start_services() { @@ -126,19 +125,27 @@ function stop_docker() { function main() { + echo "::group::stop_docker" stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi - echo "Dump current docker ps" - docker ps - start_time=$(date +%s) + echo "::endgroup::" + + echo "::group::start_services" start_services - end_time=$(date +%s) - duration=$((end_time-start_time)) - echo "Mega service start duration is $duration s" + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + echo "::group::stop_docker" stop_docker - echo y | docker system prune + echo "::endgroup::" + + docker system prune -f } diff --git a/DocIndexRetriever/tests/test_compose_on_gaudi.sh b/DocIndexRetriever/tests/test_compose_on_gaudi.sh index 4b9de5a2df..278d0dc1b4 100644 --- a/DocIndexRetriever/tests/test_compose_on_gaudi.sh +++ b/DocIndexRetriever/tests/test_compose_on_gaudi.sh @@ -21,14 +21,15 @@ function build_docker_images() { if [ ! -d "GenAIComps" ] ; then git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git fi + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s echo "Build all the images with --no-cache, check docker_image_build.log for details..." docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull redis/redis-stack:7.2.0-v9 - docker pull ghcr.io/huggingface/tei-gaudi:1.5.0 docker images && sleep 1s - echo "Docker images built!" } function start_services() { @@ -116,19 +117,27 @@ function stop_docker() { function main() { + echo "::group::stop_docker" stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi - echo "Dump current docker ps" - docker ps - start_time=$(date +%s) + echo "::endgroup::" + + echo "::group::start_services" start_services - end_time=$(date +%s) - duration=$((end_time-start_time)) - echo "Mega service start duration is $duration s" + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + echo "::group::stop_docker" stop_docker - echo y | docker system prune + echo "::endgroup::" + + docker system prune -f } diff --git a/DocIndexRetriever/tests/test_compose_on_xeon.sh b/DocIndexRetriever/tests/test_compose_on_xeon.sh index 467411653c..6d81b76c06 100644 --- a/DocIndexRetriever/tests/test_compose_on_xeon.sh +++ b/DocIndexRetriever/tests/test_compose_on_xeon.sh @@ -21,14 +21,15 @@ function build_docker_images() { if [ ! -d "GenAIComps" ] ; then git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git fi + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s + service_list="dataprep embedding retriever reranking doc-index-retriever" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6 - docker pull redis/redis-stack:7.2.0-v9 docker images && sleep 1s - - echo "Docker images built!" } function start_services() { @@ -125,20 +126,27 @@ function stop_docker() { function main() { + echo "::group::stop_docker" stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi - echo "Dump current docker ps" - docker ps + echo "::endgroup::" - start_time=$(date +%s) + echo "::group::start_services" start_services - end_time=$(date +%s) - duration=$((end_time-start_time)) - echo "Mega service start duration is $duration s" + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + echo "::group::stop_docker" stop_docker - echo y | docker system prune + echo "::endgroup::" + + docker system prune -f } diff --git a/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh b/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh index c0e32c4e93..dde5d84ef9 100644 --- a/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh +++ b/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh @@ -21,6 +21,11 @@ function build_docker_images() { if [ ! -d "GenAIComps" ] ; then git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git fi + pushd GenAIComps + echo "GenAIComps test commit is $(git rev-parse HEAD)" + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s + service_list="dataprep embedding retriever doc-index-retriever" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log @@ -114,19 +119,27 @@ function stop_docker() { function main() { + echo "::group::stop_docker" stop_docker + echo "::endgroup::" + + echo "::group::build_docker_images" if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi - echo "Dump current docker ps" - docker ps - start_time=$(date +%s) + echo "::endgroup::" + + echo "::group::start_services" start_services - end_time=$(date +%s) - duration=$((end_time-start_time)) - echo "Mega service start duration is $duration s" + echo "::endgroup::" + + echo "::group::validate_megaservice" validate_megaservice + echo "::endgroup::" + echo "::group::stop_docker" stop_docker - echo y | docker system prune + echo "::endgroup::" + + docker system prune -f }