Skip to content

Commit 3f2e7b7

Browse files
chensuyuechensuyue
authored
add image build for new examples (#802)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent 264759d commit 3f2e7b7

File tree

11 files changed

+81
-64
lines changed

11 files changed

+81
-64
lines changed

‎DocIndexRetriever/docker_image_build/build.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,9 @@ services:
3535
dockerfile: comps/dataprep/redis/langchain/Dockerfile
3636
extends: doc-index-retriever
3737
image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
38+
tei-gaudi:
39+
build:
40+
context: tei-gaudi
41+
dockerfile: Dockerfile-hpu
42+
extends: doc-index-retriever
43+
image: ${REGISTRY:-opea}/tei-gaudi:${TAG:-latest}

‎DocIndexRetriever/tests/test_compose_on_gaudi.sh

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -e
6-
echo "IMAGE_REPO=${IMAGE_REPO}"
6+
IMAGE_REPO=${IMAGE_REPO:-"opea"}
77
IMAGE_TAG=${IMAGE_TAG:-"latest"}
88
echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
99
echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
10-
# export REGISTRY=${IMAGE_REPO}
11-
# export TAG=${IMAGE_TAG}
10+
export REGISTRY=${IMAGE_REPO}
11+
export TAG=${IMAGE_TAG}
1212

1313
WORKPATH=$(dirname "$PWD")
1414
LOG_PATH="$WORKPATH/tests"
@@ -19,22 +19,18 @@ function build_docker_images() {
1919
if [ ! -d "GenAIComps" ] ; then
2020
git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
2121
fi
22-
cd GenAIComps
22+
if [ ! -d "tei-gaudi" ] ; then
23+
git clone https://github.com/huggingface/tei-gaudi
24+
fi
2325

24-
docker build -t opea/embedding-tei:latest -f comps/embeddings/tei/langchain/Dockerfile .
25-
docker build -t opea/retriever-redis:latest -f comps/retrievers/redis/langchain/Dockerfile .
26-
docker build -t opea/reranking-tei:latest -f comps/reranks/tei/Dockerfile .
27-
docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/Dockerfile .
26+
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
27+
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
2828

29-
docker pull ghcr.io/huggingface/tgi-gaudi:latest
3029
docker pull redis/redis-stack:7.2.0-v9
31-
32-
cd $WORKPATH/
33-
docker build -t opea/doc-index-retriever:latest -f ./Dockerfile .
30+
docker images && sleep 1s
3431
}
3532

3633
function start_services() {
37-
# build tei-gaudi for each test instead of pull from local registry
3834
cd $WORKPATH/docker_compose/intel/hpu/gaudi
3935
export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
4036
export RERANK_MODEL_ID="BAAI/bge-reranker-base"

‎DocIndexRetriever/tests/test_compose_on_xeon.sh

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -e
6-
echo "IMAGE_REPO=${IMAGE_REPO}"
6+
IMAGE_REPO=${IMAGE_REPO:-"opea"}
77
IMAGE_TAG=${IMAGE_TAG:-"latest"}
88
echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
99
echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
10-
# export REGISTRY=${IMAGE_REPO}
11-
# export TAG=${IMAGE_TAG}
10+
export REGISTRY=${IMAGE_REPO}
11+
export TAG=${IMAGE_TAG}
1212

1313
WORKPATH=$(dirname "$PWD")
1414
LOG_PATH="$WORKPATH/tests"
@@ -19,18 +19,12 @@ function build_docker_images() {
1919
if [ ! -d "GenAIComps" ] ; then
2020
git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
2121
fi
22-
cd GenAIComps
23-
24-
docker build -t opea/embedding-tei:latest -f comps/embeddings/tei/langchain/Dockerfile .
25-
docker build -t opea/retriever-redis:latest -f comps/retrievers/redis/langchain/Dockerfile .
26-
docker build -t opea/reranking-tei:latest -f comps/reranks/tei/Dockerfile .
27-
docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/Dockerfile .
22+
service_list="dataprep-redis embedding-tei retriever-redis reranking-tei doc-index-retriever"
23+
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
2824

2925
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
3026
docker pull redis/redis-stack:7.2.0-v9
31-
32-
cd $WORKPATH/
33-
docker build -t opea/doc-index-retriever:latest -f ./Dockerfile .
27+
docker images && sleep 1s
3428
}
3529

3630
function start_services() {

‎InstructionTuning/docker_image_build/build.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
services:
5+
finetuning:
6+
build:
7+
args:
8+
http_proxy: ${http_proxy}
9+
https_proxy: ${https_proxy}
10+
no_proxy: ${no_proxy}
11+
context: GenAIComps
12+
dockerfile: comps/finetuning/Dockerfile
13+
image: ${REGISTRY:-opea}/finetuning:${TAG:-latest}

‎InstructionTuning/tests/test_compose_on_xeon.sh

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -x
6+
IMAGE_REPO=${IMAGE_REPO:-"opea"}
7+
IMAGE_TAG=${IMAGE_TAG:-"latest"}
8+
echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
9+
echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
10+
export REGISTRY=${IMAGE_REPO}
11+
export TAG=${IMAGE_TAG}
612

713
WORKPATH=$(dirname "$PWD")
814
LOG_PATH="$WORKPATH/tests"
@@ -11,24 +17,16 @@ finetuning_service_port=8015
1117
ray_port=8265
1218

1319
function build_docker_images() {
14-
cd $WORKPATH
20+
cd $WORKPATH/docker_image_build
1521
if [ ! -d "GenAIComps" ] ; then
1622
git clone https://github.com/opea-project/GenAIComps.git
1723
fi
18-
cd GenAIComps
19-
echo PWD: $(pwd)
20-
docker build -t opea/finetuning:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg HF_TOKEN=$HF_TOKEN -f comps/finetuning/Dockerfile .
21-
if [ $? -ne 0 ]; then
22-
echo "opea/finetuning built fail"
23-
exit 1
24-
else
25-
echo "opea/finetuning built successful"
26-
fi
24+
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
2725
}
2826

2927
function start_service() {
3028
export no_proxy="localhost,127.0.0.1,"${ip_address}
31-
docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy opea/finetuning:latest
29+
docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy ${IMAGE_REPO}/finetuning:${IMAGE_TAG}
3230
sleep 1m
3331
}
3432

‎RerankFinetuning/docker_image_build/build.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
services:
5+
finetuning:
6+
build:
7+
args:
8+
http_proxy: ${http_proxy}
9+
https_proxy: ${https_proxy}
10+
no_proxy: ${no_proxy}
11+
context: GenAIComps
12+
dockerfile: comps/finetuning/Dockerfile
13+
image: ${REGISTRY:-opea}/finetuning:${TAG:-latest}

‎RerankFinetuning/tests/test_compose_on_xeon.sh

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44
set -x
5+
IMAGE_REPO=${IMAGE_REPO:-"opea"}
6+
IMAGE_TAG=${IMAGE_TAG:-"latest"}
7+
echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
8+
echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
9+
export REGISTRY=${IMAGE_REPO}
10+
export TAG=${IMAGE_TAG}
511

612
WORKPATH=$(dirname "$PWD")
713
LOG_PATH="$WORKPATH/tests"
@@ -10,24 +16,16 @@ finetuning_service_port=8015
1016
ray_port=8265
1117

1218
function build_docker_images() {
13-
cd $WORKPATH
19+
cd $WORKPATH/docker_image_build
1420
if [ ! -d "GenAIComps" ] ; then
1521
git clone https://github.com/opea-project/GenAIComps.git
1622
fi
17-
cd GenAIComps
18-
echo PWD: $(pwd)
19-
docker build -t opea/finetuning:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg HF_TOKEN=$HF_TOKEN -f comps/finetuning/Dockerfile .
20-
if [ $? -ne 0 ]; then
21-
echo "opea/finetuning built fail"
22-
exit 1
23-
else
24-
echo "opea/finetuning built successful"
25-
fi
23+
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
2624
}
2725

2826
function start_service() {
2927
export no_proxy="localhost,127.0.0.1,"${ip_address}
30-
docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy opea/finetuning:latest
28+
docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy ${IMAGE_REPO}/finetuning:${IMAGE_TAG}
3129
sleep 1m
3230
}
3331

‎VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ cd GenAIComps
5656
### 1. Build Embedding Image
5757

5858
```bash
59-
docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile .
59+
docker build -t opea/embedding-multimodal-clip:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile .
6060
```
6161

6262
### 2. Build Retriever Image
@@ -83,7 +83,7 @@ docker build -t opea/lvm-video-llama:latest --build-arg https_proxy=$https_proxy
8383
### 5. Build Dataprep Image
8484

8585
```bash
86-
docker build -t opea/dataprep-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/vdms/multimodal_langchain/Dockerfile .
86+
docker build -t opea/dataprep-multimodal-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/vdms/multimodal_langchain/Dockerfile .
8787
cd ..
8888
```
8989

@@ -110,8 +110,8 @@ docker build -t opea/videoragqna-ui:latest --build-arg https_proxy=$https_proxy
110110

111111
Then run the command `docker images`, you will have the following 8 Docker Images:
112112

113-
1. `opea/dataprep-vdms:latest`
114-
2. `opea/embedding-multimodal:latest`
113+
1. `opea/dataprep-multimodal-vdms:latest`
114+
2. `opea/embedding-multimodal-clip:latest`
115115
3. `opea/retriever-vdms:latest`
116116
4. `opea/reranking-videoragqna:latest`
117117
5. `opea/video-llama-lvm-server:latest`

‎VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ services:
1010
ports:
1111
- "8001:55555"
1212
dataprep:
13-
image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
13+
image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest}
1414
container_name: dataprep-vdms-server
1515
depends_on:
1616
- vdms-vector-db
@@ -28,7 +28,7 @@ services:
2828
- /home/$USER/.cache/clip:/home/user/.cache/clip
2929
- /home/$USER/.cache/huggingface/hub:/home/user/.cache/huggingface/hub
3030
embedding:
31-
image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
31+
image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest}
3232
container_name: embedding-multimodal-server
3333
ports:
3434
- "6000:6000"

‎VideoRAGQnA/docker_image_build/build.yaml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,43 +11,43 @@ services:
1111
context: ../
1212
dockerfile: ./Dockerfile
1313
image: ${REGISTRY:-opea}/videoragqna:${TAG:-latest}
14-
videoragqna-xeon-ui-server:
14+
videoragqna-ui:
1515
build:
1616
context: ../ui
1717
dockerfile: ./docker/Dockerfile
1818
extends: videoragqna
1919
image: ${REGISTRY:-opea}/videoragqna-ui:${TAG:-latest}
20-
dataprep:
20+
dataprep-multimodal-vdms:
2121
build:
2222
context: GenAIComps
2323
dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile
2424
extends: videoragqna
25-
image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
26-
embedding:
25+
image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest}
26+
embedding-multimodal-clip:
2727
build:
2828
context: GenAIComps
2929
dockerfile: comps/embeddings/multimodal_clip/Dockerfile
3030
extends: videoragqna
31-
image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
32-
retriever:
31+
image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest}
32+
retriever-vdms:
3333
build:
3434
context: GenAIComps
3535
dockerfile: comps/retrievers/vdms/langchain/Dockerfile
3636
extends: videoragqna
3737
image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest}
38-
reranking:
38+
reranking-videoragqna:
3939
build:
4040
context: GenAIComps
4141
dockerfile: comps/reranks/video-rag-qna/Dockerfile
4242
extends: videoragqna
4343
image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest}
44-
lvm-video-llama:
44+
video-llama-lvm-server:
4545
build:
4646
context: GenAIComps
4747
dockerfile: comps/lvms/video-llama/dependency/Dockerfile
4848
extends: videoragqna
4949
image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest}
50-
lvm:
50+
lvm-video-llama:
5151
build:
5252
context: GenAIComps
5353
dockerfile: comps/lvms/video-llama/Dockerfile

‎VideoRAGQnA/tests/test_compose_on_xeon.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ function build_docker_images() {
1919
git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
2020

2121
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
22-
service_list="videoragqna videoragqna-xeon-ui-server dataprep embedding retriever reranking lvm-video-llama lvm"
23-
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
22+
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
2423

2524
docker pull intellabs/vdms:v2.8.0
2625
docker images && sleep 1s

0 commit comments

Comments
 (0)