Skip to content

Commit 39409d7

Browse files
XinyaoWaXinyaoWa
authored
Align OpenAI API for FaqGen, DocSum (#1401)
Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
1 parent 71e3c57 commit 39409d7

File tree

10 files changed

+32
-23
lines changed

10 files changed

+32
-23
lines changed

‎DocSum/docker_compose/intel/hpu/gaudi/compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ services:
88
ports:
99
- ${LLM_ENDPOINT_PORT:-8008}:80
1010
volumes:
11-
- "./data:/data"
11+
- "${DATA_PATH:-data}:/data"
1212
environment:
1313
no_proxy: ${no_proxy}
1414
http_proxy: ${http_proxy}

‎DocSum/docsum.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
ChatCompletionResponse,
1616
ChatCompletionResponseChoice,
1717
ChatMessage,
18+
DocSumChatCompletionRequest,
1819
UsageInfo,
1920
)
20-
from comps.cores.proto.docarray import DocSumLLMParams
2121
from fastapi import File, Request, UploadFile
2222
from fastapi.responses import StreamingResponse
2323

@@ -34,14 +34,20 @@ def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **k
3434
if self.services[cur_node].service_type == ServiceType.LLM:
3535
for key_to_replace in ["text", "asr_result"]:
3636
if key_to_replace in inputs:
37-
inputs["query"] = inputs[key_to_replace]
37+
inputs["messages"] = inputs[key_to_replace]
3838
del inputs[key_to_replace]
3939

4040
docsum_parameters = kwargs.get("docsum_parameters", None)
4141
if docsum_parameters:
4242
docsum_parameters = docsum_parameters.model_dump()
43-
del docsum_parameters["query"]
43+
del docsum_parameters["messages"]
4444
inputs.update(docsum_parameters)
45+
if "id" in inputs:
46+
del inputs["id"]
47+
if "max_new_tokens" in inputs:
48+
del inputs["max_new_tokens"]
49+
if "input" in inputs:
50+
del inputs["input"]
4551
elif self.services[cur_node].service_type == ServiceType.ASR:
4652
if "video" in inputs:
4753
audio_base64 = video2audio(inputs["video"])
@@ -217,13 +223,13 @@ async def handle_request(self, request: Request, files: List[UploadFile] = File(
217223
initial_inputs_data = {}
218224
initial_inputs_data[data_type] = prompt
219225
else:
220-
initial_inputs_data = {"query": prompt}
226+
initial_inputs_data = {"messages": prompt}
221227

222228
else:
223229
raise ValueError(f"Unknown request type: {request.headers.get('content-type')}")
224230

225-
docsum_parameters = DocSumLLMParams(
226-
query="",
231+
docsum_parameters = DocSumChatCompletionRequest(
232+
messages="",
227233
max_tokens=chat_request.max_tokens if chat_request.max_tokens else 1024,
228234
top_k=chat_request.top_k if chat_request.top_k else 10,
229235
top_p=chat_request.top_p if chat_request.top_p else 0.95,

‎DocSum/tests/test_compose_on_gaudi.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ export DOCSUM_PORT=9000
2828
export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"
2929
export DocSum_COMPONENT_NAME="OpeaDocSumTgi"
3030
export LOGFLAG=True
31+
export DATA_PATH="/data/cache"
3132

3233
WORKPATH=$(dirname "$PWD")
3334
LOG_PATH="$WORKPATH/tests"
@@ -51,6 +52,7 @@ function build_docker_images() {
5152
function start_services() {
5253
cd $WORKPATH/docker_compose/intel/hpu/gaudi
5354

55+
sed -i "s|container_name: docsum-gaudi-backend-server|container_name: docsum-gaudi-backend-server\n volumes:\n - \"${WORKPATH}\/docker_image_build\/GenAIComps:\/home\/user\/GenAIComps\"|g" compose.yaml
5456
docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
5557
sleep 3m
5658
}
@@ -158,10 +160,10 @@ function validate_microservices() {
158160
# llm microservice
159161
validate_services_json \
160162
"${host_ip}:9000/v1/docsum" \
161-
"data: " \
163+
"text" \
162164
"llm-docsum-tgi" \
163165
"llm-docsum-gaudi-server" \
164-
'{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
166+
'{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
165167

166168
# whisper microservice
167169
ulimit -s 65536

‎DocSum/tests/test_compose_on_rocm.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ function build_docker_images() {
5050
function start_services() {
5151
cd "$WORKPATH"/docker_compose/amd/gpu/rocm
5252
sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env
53-
53+
sed -i "s|container_name: docsum-backend-server|container_name: docsum-backend-server\n volumes:\n - \"${WORKPATH}\/docker_image_build\/GenAIComps:\/home\/user\/GenAIComps\"|g" compose.yaml
5454
# Start Docker Containers
5555
docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log
5656
sleep 3m
@@ -138,10 +138,10 @@ function validate_microservices() {
138138
# llm microservice
139139
validate_services \
140140
"${host_ip}:9000/v1/docsum" \
141-
"data: " \
141+
"text" \
142142
"docsum-llm-server" \
143143
"docsum-llm-server" \
144-
'{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
144+
'{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
145145

146146
}
147147

‎DocSum/tests/test_compose_on_xeon.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ function build_docker_images() {
4949

5050
function start_services() {
5151
cd $WORKPATH/docker_compose/intel/cpu/xeon/
52-
52+
sed -i "s|container_name: docsum-xeon-backend-server|container_name: docsum-xeon-backend-server\n volumes:\n - \"${WORKPATH}\/docker_image_build\/GenAIComps:\/home\/user\/GenAIComps\"|g" compose.yaml
5353
docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
5454
sleep 3m
5555
}
@@ -160,10 +160,10 @@ function validate_microservices() {
160160
# llm microservice
161161
validate_services_json \
162162
"${host_ip}:9000/v1/docsum" \
163-
"data: " \
163+
"text" \
164164
"llm-docsum-tgi" \
165165
"llm-docsum-server" \
166-
'{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
166+
'{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
167167

168168
# whisper microservice
169169
ulimit -s 65536

‎FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ services:
88
ports:
99
- ${LLM_ENDPOINT_PORT:-8008}:80
1010
volumes:
11-
- "./data:/data"
11+
- "${DATA_PATH:-data}:/data"
1212
environment:
1313
no_proxy: ${no_proxy}
1414
http_proxy: ${http_proxy}

‎FaqGen/faqgen.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ async def handle_request(self, request: Request, files: List[UploadFile] = File(
113113
model=chat_request.model if chat_request.model else None,
114114
)
115115
result_dict, runtime_graph = await self.megaservice.schedule(
116-
initial_inputs={"query": prompt}, llm_parameters=parameters
116+
initial_inputs={"messages": prompt}, llm_parameters=parameters
117117
)
118118
for node, response in result_dict.items():
119119
# Here it suppose the last microservice in the megaservice is LLM.

‎FaqGen/tests/test_compose_on_gaudi.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ export TAG=${IMAGE_TAG}
1313
WORKPATH=$(dirname "$PWD")
1414
LOG_PATH="$WORKPATH/tests"
1515
ip_address=$(hostname -I | awk '{print $1}')
16+
export DATA_PATH="/data/cache"
1617

1718
function build_docker_images() {
1819
cd $WORKPATH/docker_image_build
@@ -91,10 +92,10 @@ function validate_microservices() {
9192
# llm microservice
9293
validate_services \
9394
"${ip_address}:9000/v1/faqgen" \
94-
"data: " \
95+
"text" \
9596
"llm" \
9697
"llm-faqgen-server" \
97-
'{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
98+
'{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
9899
}
99100

100101
function validate_megaservice() {

‎FaqGen/tests/test_compose_on_rocm.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,10 @@ function validate_microservices() {
9595
# llm microservice
9696
validate_services \
9797
"${ip_address}:9000/v1/faqgen" \
98-
"data: " \
98+
"text" \
9999
"llm" \
100100
"faqgen-llm-server" \
101-
'{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
101+
'{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
102102
}
103103

104104
function validate_megaservice() {

‎FaqGen/tests/test_compose_on_xeon.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,10 @@ function validate_microservices() {
9191
# llm microservice
9292
validate_services \
9393
"${ip_address}:9000/v1/faqgen" \
94-
"data: " \
94+
"text" \
9595
"llm" \
9696
"llm-faqgen-server" \
97-
'{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
97+
'{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
9898
}
9999

100100
function validate_megaservice() {

0 commit comments

Comments
 (0)