Skip to content

Commit 4f3fdf1

Browse files
oobaboogaoobabooga
authored
Merge pull request oobabooga#5404 from oobabooga/dev
Merge dev branch
2 parents e7a760e + 167ee72 commit 4f3fdf1

25 files changed

+336
-235
lines changed

‎Colab-TextGen-GPU.ipynb

+2-5
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,9 @@
6565
" torver = torch.__version__\n",
6666
" print(f\"TORCH: {torver}\")\n",
6767
" is_cuda118 = '+cu118' in torver # 2.1.0+cu118\n",
68-
" is_cuda117 = '+cu117' in torver # 2.0.1+cu117\n",
6968
"\n",
7069
" textgen_requirements = open('requirements.txt').read().splitlines()\n",
71-
" if is_cuda117:\n",
72-
" textgen_requirements = [req.replace('+cu121', '+cu117').replace('+cu122', '+cu117').replace('torch2.1', 'torch2.0') for req in textgen_requirements]\n",
73-
" elif is_cuda118:\n",
70+
" if is_cuda118:\n",
7471
" textgen_requirements = [req.replace('+cu121', '+cu118').replace('+cu122', '+cu118') for req in textgen_requirements]\n",
7572
" with open('temp_requirements.txt', 'w') as file:\n",
7673
" file.write('\\n'.join(textgen_requirements))\n",
@@ -130,4 +127,4 @@
130127
"outputs": []
131128
}
132129
]
133-
}
130+
}

‎README.md

+12-1
Original file line numberDiff line numberDiff line change
@@ -160,12 +160,23 @@ The `requirements*.txt` above contain various wheels precompiled through GitHub
160160
### Alternative: Docker
161161

162162
```
163-
ln -s docker/{nvidia/Dockerfile,docker-compose.yml,.dockerignore} .
163+
For NVIDIA GPU:
164+
ln -s docker/{nvidia/Dockerfile,nvidia/docker-compose.yml,.dockerignore} .
165+
For AMD GPU:
166+
ln -s docker/{amd/Dockerfile,intel/docker-compose.yml,.dockerignore} .
167+
For Intel GPU:
168+
ln -s docker/{intel/Dockerfile,amd/docker-compose.yml,.dockerignore} .
169+
For CPU only
170+
ln -s docker/{cpu/Dockerfile,cpu/docker-compose.yml,.dockerignore} .
164171
cp docker/.env.example .env
172+
#Create logs/cache dir :
173+
mkdir -p logs cache
165174
# Edit .env and set:
166175
# TORCH_CUDA_ARCH_LIST based on your GPU model
167176
# APP_RUNTIME_GID your host user's group id (run `id -g` in a terminal)
168177
# BUILD_EXTENIONS optionally add comma separated list of extensions to build
178+
# Edit CMD_FLAGS.txt and add in it the options you want to execute (like --listen --cpu)
179+
#
169180
docker compose up --build
170181
```
171182

‎docker/.env.example

+3
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,6 @@ APP_RUNTIME_GID=6972
2020
# override default app build permissions (handy for deploying to cloud)
2121
#APP_GID=6972
2222
#APP_UID=6972
23+
# Set cache env
24+
TRANSFORMERS_CACHE=/home/app/text-generation-webui/cache/
25+
HF_HOME=/home/app/text-generation-webui/cache/

‎docker/amd/Dockerfile

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# BUILDER
2+
FROM ubuntu:22.04
3+
WORKDIR /builder
4+
ARG TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-3.5;5.0;6.0;6.1;7.0;7.5;8.0;8.6+PTX}"
5+
ARG BUILD_EXTENSIONS="${BUILD_EXTENSIONS:-}"
6+
ARG APP_UID="${APP_UID:-6972}"
7+
ARG APP_GID="${APP_GID:-6972}"
8+
9+
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,rw \
10+
apt update && \
11+
apt install --no-install-recommends -y git vim build-essential python3-dev pip bash curl && \
12+
rm -rf /var/lib/apt/lists/*
13+
WORKDIR /home/app/
14+
RUN git clone https://github.com/oobabooga/text-generation-webui.git
15+
WORKDIR /home/app/text-generation-webui
16+
RUN GPU_CHOICE=B USE_CUDA118=FALSE LAUNCH_AFTER_INSTALL=FALSE INSTALL_EXTENSIONS=TRUE ./start_linux.sh --verbose
17+
COPY CMD_FLAGS.txt /home/app/text-generation-webui/
18+
EXPOSE ${CONTAINER_PORT:-7860} ${CONTAINER_API_PORT:-5000} ${CONTAINER_API_STREAM_PORT:-5005}
19+
WORKDIR /home/app/text-generation-webui
20+
# set umask to ensure group read / write at runtime
21+
CMD umask 0002 && export HOME=/home/app/text-generation-webui && ./start_linux.sh

‎docker/amd/docker-compose.yml

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
version: "3.3"
2+
services:
3+
text-generation-webui:
4+
build:
5+
context: .
6+
args:
7+
# Requirements file to use:
8+
# | GPU | CPU | requirements file to use |
9+
# |--------|---------|---------|
10+
# | NVIDIA | has AVX2 | `requirements.txt` |
11+
# | NVIDIA | no AVX2 | `requirements_noavx2.txt` |
12+
# | AMD | has AVX2 | `requirements_amd.txt` |
13+
# | AMD | no AVX2 | `requirements_amd_noavx2.txt` |
14+
# | CPU only | has AVX2 | `requirements_cpu_only.txt` |
15+
# | CPU only | no AVX2 | `requirements_cpu_only_noavx2.txt` |
16+
# | Apple | Intel | `requirements_apple_intel.txt` |
17+
# | Apple | Apple Silicon | `requirements_apple_silicon.txt` |
18+
# Default: requirements.txt`
19+
# BUILD_REQUIREMENTS: requirements.txt
20+
21+
# Extension requirements to build:
22+
# BUILD_EXTENSIONS:
23+
24+
# specify which cuda version your card supports: https://developer.nvidia.com/cuda-gpus
25+
TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5}
26+
BUILD_EXTENSIONS: ${BUILD_EXTENSIONS:-}
27+
APP_GID: ${APP_GID:-6972}
28+
APP_UID: ${APP_UID-6972}
29+
env_file: .env
30+
user: "${APP_RUNTIME_UID:-6972}:${APP_RUNTIME_GID:-6972}"
31+
ports:
32+
- "${HOST_PORT:-7860}:${CONTAINER_PORT:-7860}"
33+
- "${HOST_API_PORT:-5000}:${CONTAINER_API_PORT:-5000}"
34+
stdin_open: true
35+
group_add:
36+
- video
37+
tty: true
38+
ipc: host
39+
devices:
40+
- /dev/kfd
41+
- /dev/dri
42+
cap_add:
43+
- SYS_PTRACE
44+
security_opt:
45+
- seccomp=unconfined
46+
volumes:
47+
- ./cache:/home/app/text-generation-webui/cache
48+
- ./characters:/home/app/text-generation-webui/characters
49+
- ./extensions:/home/app/text-generation-webui/extensions
50+
- ./loras:/home/app/text-generation-webui/loras
51+
- ./logs:/home/app/text-generation-webui/logs
52+
- ./models:/home/app/text-generation-webui/models
53+
- ./presets:/home/app/text-generation-webui/presets
54+
- ./prompts:/home/app/text-generation-webui/prompts
55+
- ./softprompts:/home/app/text-generation-webui/softprompts
56+
- ./training:/home/app/text-generation-webui/training
57+
- ./cloudflared:/etc/cloudflared

‎docker/cpu/Dockerfile

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# BUILDER
2+
FROM ubuntu:22.04
3+
WORKDIR /builder
4+
ARG TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-3.5;5.0;6.0;6.1;7.0;7.5;8.0;8.6+PTX}"
5+
ARG BUILD_EXTENSIONS="${BUILD_EXTENSIONS:-}"
6+
ARG APP_UID="${APP_UID:-6972}"
7+
ARG APP_GID="${APP_GID:-6972}"
8+
ARG GPU_CHOICE=A
9+
ARG USE_CUDA118=FALSE
10+
ARG LAUNCH_AFTER_INSTALL=FALSE
11+
ARG INSTALL_EXTENSIONS=TRUE
12+
13+
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,rw \
14+
apt update && \
15+
apt install --no-install-recommends -y git vim build-essential python3-dev pip bash curl && \
16+
rm -rf /var/lib/apt/lists/*
17+
WORKDIR /home/app/
18+
RUN git clone https://github.com/oobabooga/text-generation-webui.git
19+
WORKDIR /home/app/text-generation-webui
20+
RUN GPU_CHOICE=N USE_CUDA118=FALSE LAUNCH_AFTER_INSTALL=FALSE INSTALL_EXTENSIONS=TRUE ./start_linux.sh --verbose
21+
COPY CMD_FLAGS.txt /home/app/text-generation-webui/
22+
EXPOSE ${CONTAINER_PORT:-7860} ${CONTAINER_API_PORT:-5000} ${CONTAINER_API_STREAM_PORT:-5005}
23+
# set umask to ensure group read / write at runtime
24+
WORKDIR /home/app/text-generation-webui
25+
CMD umask 0002 && export HOME=/home/app/text-generation-webui && ./start_linux.sh

‎docker/cpu/docker-compose.yml

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
version: "3.3"
2+
services:
3+
text-generation-webui:
4+
build:
5+
context: .
6+
args:
7+
# Requirements file to use:
8+
# | GPU | CPU | requirements file to use |
9+
# |--------|---------|---------|
10+
# | NVIDIA | has AVX2 | `requirements.txt` |
11+
# | NVIDIA | no AVX2 | `requirements_noavx2.txt` |
12+
# | AMD | has AVX2 | `requirements_amd.txt` |
13+
# | AMD | no AVX2 | `requirements_amd_noavx2.txt` |
14+
# | CPU only | has AVX2 | `requirements_cpu_only.txt` |
15+
# | CPU only | no AVX2 | `requirements_cpu_only_noavx2.txt` |
16+
# | Apple | Intel | `requirements_apple_intel.txt` |
17+
# | Apple | Apple Silicon | `requirements_apple_silicon.txt` |
18+
# Default: requirements.txt`
19+
# BUILD_REQUIREMENTS: requirements.txt
20+
21+
# Extension requirements to build:
22+
# BUILD_EXTENSIONS:
23+
24+
# specify which cuda version your card supports: https://developer.nvidia.com/cuda-gpus
25+
TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5}
26+
BUILD_EXTENSIONS: ${BUILD_EXTENSIONS:-}
27+
APP_GID: ${APP_GID:-6972}
28+
APP_UID: ${APP_UID-6972}
29+
env_file: .env
30+
user: "${APP_RUNTIME_UID:-6972}:${APP_RUNTIME_GID:-6972}"
31+
ports:
32+
- "${HOST_PORT:-7860}:${CONTAINER_PORT:-7860}"
33+
- "${HOST_API_PORT:-5000}:${CONTAINER_API_PORT:-5000}"
34+
stdin_open: true
35+
tty: true
36+
volumes:
37+
- ./cache:/home/app/text-generation-webui/cache
38+
- ./characters:/home/app/text-generation-webui/characters
39+
- ./extensions:/home/app/text-generation-webui/extensions
40+
- ./loras:/home/app/text-generation-webui/loras
41+
- ./logs:/home/app/text-generation-webui/logs
42+
- ./models:/home/app/text-generation-webui/models
43+
- ./presets:/home/app/text-generation-webui/presets
44+
- ./prompts:/home/app/text-generation-webui/prompts
45+
- ./softprompts:/home/app/text-generation-webui/softprompts
46+
- ./training:/home/app/text-generation-webui/training
47+
- ./cloudflared:/etc/cloudflared

‎docker/intel/Dockerfile

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# BUILDER
2+
FROM ubuntu:22.04
3+
WORKDIR /builder
4+
ARG TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-3.5;5.0;6.0;6.1;7.0;7.5;8.0;8.6+PTX}"
5+
ARG BUILD_EXTENSIONS="${BUILD_EXTENSIONS:-}"
6+
ARG APP_UID="${APP_UID:-6972}"
7+
ARG APP_GID="${APP_GID:-6972}"
8+
9+
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,rw \
10+
apt update && \
11+
apt install --no-install-recommends -y git vim build-essential python3-dev pip bash curl && \
12+
rm -rf /var/lib/apt/lists/*
13+
WORKDIR /home/app/
14+
RUN git clone https://github.com/oobabooga/text-generation-webui.git
15+
WORKDIR /home/app/text-generation-webui
16+
RUN GPU_CHOICE=D USE_CUDA118=FALSE LAUNCH_AFTER_INSTALL=FALSE INSTALL_EXTENSIONS=TRUE ./start_linux.sh --verbose
17+
COPY CMD_FLAGS.txt /home/app/text-generation-webui/
18+
EXPOSE ${CONTAINER_PORT:-7860} ${CONTAINER_API_PORT:-5000} ${CONTAINER_API_STREAM_PORT:-5005}
19+
# set umask to ensure group read / write at runtime
20+
WORKDIR /home/app/text-generation-webui
21+
CMD umask 0002 && export HOME=/home/app/text-generation-webui && ./start_linux.sh

‎docker/intel/docker-compose.yml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
version: "3.3"
2+
services:
3+
text-generation-webui:
4+
build:
5+
context: .
6+
args:
7+
# Requirements file to use:
8+
# | GPU | CPU | requirements file to use |
9+
# |--------|---------|---------|
10+
# | NVIDIA | has AVX2 | `requirements.txt` |
11+
# | NVIDIA | no AVX2 | `requirements_noavx2.txt` |
12+
# | AMD | has AVX2 | `requirements_amd.txt` |
13+
# | AMD | no AVX2 | `requirements_amd_noavx2.txt` |
14+
# | CPU only | has AVX2 | `requirements_cpu_only.txt` |
15+
# | CPU only | no AVX2 | `requirements_cpu_only_noavx2.txt` |
16+
# | Apple | Intel | `requirements_apple_intel.txt` |
17+
# | Apple | Apple Silicon | `requirements_apple_silicon.txt` |
18+
# Default: requirements.txt`
19+
# BUILD_REQUIREMENTS: requirements.txt
20+
21+
# Extension requirements to build:
22+
# BUILD_EXTENSIONS:
23+
24+
# specify which cuda version your card supports: https://developer.nvidia.com/cuda-gpus
25+
TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5}
26+
BUILD_EXTENSIONS: ${BUILD_EXTENSIONS:-}
27+
APP_GID: ${APP_GID:-6972}
28+
APP_UID: ${APP_UID-6972}
29+
env_file: .env
30+
user: "${APP_RUNTIME_UID:-6972}:${APP_RUNTIME_GID:-6972}"
31+
ports:
32+
- "${HOST_PORT:-7860}:${CONTAINER_PORT:-7860}"
33+
- "${HOST_API_PORT:-5000}:${CONTAINER_API_PORT:-5000}"
34+
stdin_open: true
35+
group_add:
36+
- video
37+
tty: true
38+
ipc: host
39+
devices:
40+
- /dev/kfd
41+
- /dev/dri
42+
cap_add:
43+
- SYS_PTRACE
44+
security_opt:
45+
- seccomp=unconfined
46+
volumes:
47+
- ./characters:/home/app/text-generation-webui/characters
48+
- ./extensions:/home/app/text-generation-webui/extensions
49+
- ./loras:/home/app/text-generation-webui/loras
50+
- ./models:/home/app/text-generation-webui/models
51+
- ./presets:/home/app/text-generation-webui/presets
52+
- ./prompts:/home/app/text-generation-webui/prompts
53+
- ./softprompts:/home/app/text-generation-webui/softprompts
54+
- ./training:/home/app/text-generation-webui/training
55+
- ./cloudflared:/etc/cloudflared

‎docker/nvidia/Dockerfile

+9-45
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,21 @@
11
# BUILDER
2-
FROM nvidia/cuda:12.1.1-devel-ubuntu22.04 as builder
2+
FROM ubuntu:22.04
33
WORKDIR /builder
44
ARG TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-3.5;5.0;6.0;6.1;7.0;7.5;8.0;8.6+PTX}"
55
ARG BUILD_EXTENSIONS="${BUILD_EXTENSIONS:-}"
6-
ARG BUILD_REQUIREMENTS="${BUILD_REQUIREMENTS:-requirements.txt}"
76
ARG APP_UID="${APP_UID:-6972}"
87
ARG APP_GID="${APP_GID:-6972}"
9-
# create / update build env
10-
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,rw \
11-
apt update && \
12-
apt install --no-install-recommends -y git vim build-essential python3-dev pip && \
13-
rm -rf /var/lib/apt/lists/*
14-
RUN --mount=type=cache,target=/root/.cache/pip,rw \
15-
pip3 install --global --upgrade pip wheel setuptools && \
16-
# make shared builder & runtime app user
17-
addgroup --gid $APP_GID app_grp && \
18-
useradd -m -u $APP_UID --gid app_grp app
19-
USER app:app_grp
20-
# build wheels for runtime
21-
WORKDIR /home/app/build
22-
COPY --chown=app:app_grp "$BUILD_REQUIREMENTS" /home/app/build/requirements.txt
23-
COPY --chown=app:app_grp extensions /home/app/build/extensions
24-
RUN --mount=type=cache,target=/root/.cache/pip,rw \
25-
# build all requirements files as wheel dists
26-
pip3 wheel -w wheels -r requirements.txt `echo "$BUILD_EXTENSIONS" | sed -r 's/([^,]+)\s*,?\s*/ -r \/home\/app\/build\/extensions\/\1\/requirements.txt/g'`
27-
# drop wheel and setuptools .whl to avoid install issues
28-
RUN rm wheels/setuptools*.whl
298

30-
# RUNTIME
31-
FROM nvidia/cuda:12.1.1-runtime-ubuntu22.04
32-
ARG TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-3.5;5.0;6.0;6.1;7.0;7.5;8.0;8.6}"
33-
ARG APP_UID="${APP_UID:-6972}"
34-
ARG APP_GID="${APP_GID:-6972}"
35-
ENV CLI_ARGS=""
36-
# create / update runtime env
379
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,rw \
3810
apt update && \
39-
apt install --no-install-recommends -y git python3 pip && \
40-
rm -rf /var/lib/apt/lists/* && \
41-
pip3 install --global --no-cache --upgrade pip wheel setuptools && \
42-
# make shared builder & runtime app user
43-
addgroup --gid $APP_GID app_grp && \
44-
useradd -m -u $APP_UID --gid app_grp app
45-
USER app:app_grp
46-
# install locally built wheels for app
47-
WORKDIR /home/app/wheels
48-
COPY --from=builder /home/app/build/wheels /home/app/wheels
49-
COPY --chown=app:app_grp . /home/app/text-generation-webui
50-
RUN umask 0002 && \
51-
chmod g+rwX /home/app/text-generation-webui && \
52-
pip3 install --global --no-build-isolation --no-cache --no-index ./*.whl && \
53-
rm -r /home/app/wheels
11+
apt install --no-install-recommends -y git vim build-essential python3-dev pip bash curl && \
12+
rm -rf /var/lib/apt/lists/*
13+
WORKDIR /home/app/
14+
RUN git clone https://github.com/oobabooga/text-generation-webui.git
5415
WORKDIR /home/app/text-generation-webui
16+
RUN GPU_CHOICE=A USE_CUDA118=FALSE LAUNCH_AFTER_INSTALL=FALSE INSTALL_EXTENSIONS=TRUE ./start_linux.sh --verbose
17+
COPY CMD_FLAGS.txt /home/app/text-generation-webui/
5518
EXPOSE ${CONTAINER_PORT:-7860} ${CONTAINER_API_PORT:-5000} ${CONTAINER_API_STREAM_PORT:-5005}
19+
WORKDIR /home/app/text-generation-webui
5620
# set umask to ensure group read / write at runtime
57-
CMD umask 0002 && export HOME=/home/app && python3 server.py ${CLI_ARGS}
21+
CMD umask 0002 && export HOME=/home/app/text-generation-webui && ./start_linux.sh

‎docker/docker-compose.yml ‎docker/nvidia/docker-compose.yml

+2
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,11 @@ services:
3434
stdin_open: true
3535
tty: true
3636
volumes:
37+
- ./cache:/home/app/text-generation-webui/cache
3738
- ./characters:/home/app/text-generation-webui/characters
3839
- ./extensions:/home/app/text-generation-webui/extensions
3940
- ./loras:/home/app/text-generation-webui/loras
41+
- ./logs:/home/app/text-generation-webui/logs
4042
- ./models:/home/app/text-generation-webui/models
4143
- ./presets:/home/app/text-generation-webui/presets
4244
- ./prompts:/home/app/text-generation-webui/prompts

‎download-model.py

+10-2
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,18 @@ def __init__(self, max_retries=5):
3030
if max_retries:
3131
self.session.mount('https://cdn-lfs.huggingface.co', HTTPAdapter(max_retries=max_retries))
3232
self.session.mount('https://huggingface.co', HTTPAdapter(max_retries=max_retries))
33+
3334
if os.getenv('HF_USER') is not None and os.getenv('HF_PASS') is not None:
3435
self.session.auth = (os.getenv('HF_USER'), os.getenv('HF_PASS'))
35-
if os.getenv('HF_TOKEN') is not None:
36-
self.session.headers = {'authorization': f'Bearer {os.getenv("HF_TOKEN")}'}
36+
37+
try:
38+
from huggingface_hub import get_token
39+
token = get_token()
40+
except ImportError:
41+
token = os.getenv("HF_TOKEN")
42+
43+
if token is not None:
44+
self.session.headers = {'authorization': f'Bearer {token}'}
3745

3846
def sanitize_model_and_branch_names(self, model, branch):
3947
if model[-1] == '/':

0 commit comments

Comments
 (0)