Skip to content

[CICD enhance] DocIndexRetriever run CI with latest base image, group logs in GHA outputs. #1932

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 16, 2025
Merged

[CICD enhance] DocIndexRetriever run CI with latest base image, group logs in GHA outputs. #1932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion DocIndexRetriever/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

ARG IMAGE_REPO=opea
ARG BASE_TAG=latest
FROM opea/comps-base:$BASE_TAG
FROM $IMAGE_REPO/comps-base:$BASE_TAG

COPY ./retrieval_tool.py $HOME/retrieval_tool.py

Expand Down
2 changes: 2 additions & 0 deletions DocIndexRetriever/docker_image_build/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ services:
doc-index-retriever:
build:
args:
IMAGE_REPO: ${REGISTRY:-opea}
BASE_TAG: ${TAG:-latest}
http_proxy: ${http_proxy}
https_proxy: ${https_proxy}
no_proxy: ${no_proxy}
Expand Down
34 changes: 20 additions & 14 deletions DocIndexRetriever/tests/test_compose_milvus_on_gaudi.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s

service_list="dataprep embedding retriever reranking doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
docker pull quay.io/coreos/etcd:v3.5.5
docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z
docker pull milvusdb/milvus:v2.4.6
docker images && sleep 1s

echo "Docker images built!"
}

function start_services() {
Expand Down Expand Up @@ -127,19 +125,27 @@ function stop_docker() {

function main() {

echo "::group::stop_docker"
stop_docker
echo "::endgroup::"

echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
echo "::endgroup::"

echo "::group::start_services"
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
echo "::endgroup::"

echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"

echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"

docker system prune -f

}

Expand Down
33 changes: 20 additions & 13 deletions DocIndexRetriever/tests/test_compose_milvus_on_xeon.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s

service_list="dataprep embedding retriever reranking doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
docker pull quay.io/coreos/etcd:v3.5.5
docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z
docker pull milvusdb/milvus:v2.4.6
docker images && sleep 1s

echo "Docker images built!"
}

function start_services() {
Expand Down Expand Up @@ -126,19 +125,27 @@ function stop_docker() {

function main() {

echo "::group::stop_docker"
stop_docker
echo "::endgroup::"

echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
echo "::endgroup::"

echo "::group::start_services"
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
echo "::endgroup::"

echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"

echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"

docker system prune -f

}

Expand Down
29 changes: 19 additions & 10 deletions DocIndexRetriever/tests/test_compose_on_gaudi.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s

echo "Build all the images with --no-cache, check docker_image_build.log for details..."
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log

docker pull redis/redis-stack:7.2.0-v9
docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
docker images && sleep 1s
echo "Docker images built!"
}

function start_services() {
Expand Down Expand Up @@ -116,19 +117,27 @@ function stop_docker() {

function main() {

echo "::group::stop_docker"
stop_docker
echo "::endgroup::"

echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
echo "::endgroup::"

echo "::group::start_services"
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
echo "::endgroup::"

echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"

echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"

docker system prune -f

}

Expand Down
30 changes: 19 additions & 11 deletions DocIndexRetriever/tests/test_compose_on_xeon.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s

service_list="dataprep embedding retriever reranking doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
docker pull redis/redis-stack:7.2.0-v9
docker images && sleep 1s

echo "Docker images built!"
}

function start_services() {
Expand Down Expand Up @@ -125,20 +126,27 @@ function stop_docker() {

function main() {

echo "::group::stop_docker"
stop_docker
echo "::endgroup::"

echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
echo "::endgroup::"

start_time=$(date +%s)
echo "::group::start_services"
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
echo "::endgroup::"

echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"

echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"

docker system prune -f

}

Expand Down
27 changes: 20 additions & 7 deletions DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s

service_list="dataprep embedding retriever doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

Expand Down Expand Up @@ -114,19 +119,27 @@ function stop_docker() {

function main() {

echo "::group::stop_docker"
stop_docker
echo "::endgroup::"

echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
echo "::endgroup::"

echo "::group::start_services"
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
echo "::endgroup::"

echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"

echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"

docker system prune -f

}

Expand Down