Skip to content

Commit c0880b5

Browse files
oobaboogaoobabooga
committed
Merge remote-tracking branch 'turboderp/master'
2 parents 31af60a + 1a80d38 commit c0880b5

15 files changed

+546
-68
lines changed

‎.github/workflows/build-wheels-release-rocm62.yml

+217
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
name: Build Wheels & Release ROCm62
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
release:
7+
description: 'Release? 1 = yes, 0 = no'
8+
default: '0'
9+
required: true
10+
type: string
11+
12+
permissions:
13+
contents: write
14+
15+
jobs:
16+
build_wheels:
17+
name: ${{ matrix.os }} P${{ matrix.pyver }} C${{ matrix.cuda }} R${{ matrix.rocm }} T${{ matrix.torch }}
18+
runs-on: ${{ matrix.os }}
19+
strategy:
20+
matrix:
21+
include:
22+
23+
# Ubuntu 20.04 CUDA
24+
25+
# ROCm 6.2
26+
- { artname: 'wheel', os: ubuntu-20.04-l, pyver: '3.10', cuda: '', rocm: '6.2', torch: '2.5.0', cudaarch: '' }
27+
- { artname: 'wheel', os: ubuntu-20.04-l, pyver: '3.11', cuda: '', rocm: '6.2', torch: '2.5.0', cudaarch: '' }
28+
- { artname: 'wheel', os: ubuntu-20.04-l, pyver: '3.12', cuda: '', rocm: '6.2', torch: '2.5.0', cudaarch: '' }
29+
30+
# ROCm 6.2.4
31+
- { artname: 'wheel', os: ubuntu-20.04-l, pyver: '3.10', cuda: '', rocm: '6.2.4', torch: '2.6.0', cudaarch: '' }
32+
- { artname: 'wheel', os: ubuntu-20.04-l, pyver: '3.11', cuda: '', rocm: '6.2.4', torch: '2.6.0', cudaarch: '' }
33+
- { artname: 'wheel', os: ubuntu-20.04-l, pyver: '3.12', cuda: '', rocm: '6.2.4', torch: '2.6.0', cudaarch: '' }
34+
- { artname: 'wheel', os: ubuntu-20.04-l, pyver: '3.13', cuda: '', rocm: '6.2.4', torch: '2.6.0', cudaarch: '' }
35+
36+
fail-fast: false
37+
38+
defaults:
39+
run:
40+
shell: pwsh
41+
42+
steps:
43+
# Free disk space
44+
45+
- name: Free Disk Space
46+
uses: jlumbroso/free-disk-space@v1.3.1
47+
if: runner.os == 'Linux'
48+
with:
49+
tool-cache: true
50+
android: true
51+
dotnet: true
52+
haskell: true
53+
large-packages: false
54+
swap-storage: true
55+
56+
# Setup Python
57+
58+
- uses: actions/checkout@v4
59+
60+
- uses: actions/setup-python@v5.4.0
61+
with:
62+
python-version: ${{ matrix.pyver }}
63+
64+
# Get version string from package
65+
66+
- name: Get version string
67+
id: package_version
68+
run: |
69+
$versionString = Get-Content $(Join-Path 'exllamav2' 'version.py') -raw
70+
if ($versionString -match '__version__ = "(\d+\.(?:\d+\.?(?:dev\d+)?)*)"')
71+
{
72+
Write-Output $('::notice file=build-wheels-release.yml,line=200,title=Package Version::Detected package version is: {0}' -f $Matches[1])
73+
Write-Output "PACKAGE_VERSION=$($Matches[1])" >> "$env:GITHUB_OUTPUT"
74+
}
75+
else
76+
{
77+
Write-Output '::error file=build-wheels-release.yml,line=203::Could not parse version from exllamav2/version.py! You must upload wheels manually!'
78+
Write-Output "PACKAGE_VERSION=None" >> "$env:GITHUB_OUTPUT"
79+
}
80+
81+
# Pin VS build tools to 17.9 so builds won't fail
82+
83+
- name: Install VS2022 BuildTools 17.9.7
84+
run: choco install -y visualstudio2022buildtools --version=117.9.7.0 --params "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --installChannelUri https://aka.ms/vs/17/release/180911598_-255012421/channel"
85+
if: runner.os == 'Windows'
86+
87+
# Install ROCm SDK, apparently needs to happen before setting up Python
88+
89+
- name: Build for ROCm
90+
if: matrix.rocm != ''
91+
shell: bash
92+
run: |
93+
# --- Install ROCm SDK
94+
95+
export ROCM_VERSION=${{ matrix.rocm }}
96+
export TORCH_VERSION=${{ matrix.torch }}
97+
98+
[ ! -d /etc/apt/keyrings ] && sudo mkdir --parents --mode=0755 /etc/apt/keyrings
99+
wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
100+
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
101+
echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
102+
103+
sudo apt update
104+
sudo apt install rocm-hip-sdk -y
105+
sudo apt clean -y
106+
107+
echo "/opt/rocm/bin" >> $GITHUB_PATH
108+
echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV
109+
echo "ROCM_VERSION=$ROCM_VERSION" >> $GITHUB_ENV
110+
echo "USE_ROCM=1" >> $GITHUB_ENV
111+
112+
# --- Install dependencies
113+
114+
python3 -m ensurepip --upgrade
115+
pip3 install torch==${{ matrix.torch }} --index-url="https://download.pytorch.org/whl/rocm$ROCM_VERSION"
116+
pip3 install --upgrade setuptools==69.5.1 build wheel safetensors sentencepiece ninja
117+
pip3 cache purge
118+
119+
# --- Build wheel
120+
121+
python3 -m build -n --wheel -C--build-option=egg_info "-C--build-option=--tag-build=+rocm${{ matrix.rocm }}-torch${{ matrix.torch }}"
122+
123+
# Build for CUDA
124+
125+
- name: Setup Mamba
126+
if: matrix.cuda != ''
127+
uses: conda-incubator/setup-miniconda@v3.1.0
128+
with:
129+
activate-environment: "exllama"
130+
python-version: ${{ matrix.pyver }}
131+
# miniforge-variant: Mambaforge
132+
miniforge-version: latest
133+
# use-mamba: true
134+
add-pip-as-python-dependency: true
135+
auto-activate-base: false
136+
137+
- name: Build for CUDA
138+
if: matrix.cuda != ''
139+
run: |
140+
# --- Spawn the VS shell
141+
if ($IsWindows) {
142+
Import-Module 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
143+
Enter-VsDevShell -VsInstallPath 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools' -DevCmdArguments '-arch=x64 -host_arch=x64'
144+
$env:DISTUTILS_USE_SDK=1
145+
}
146+
147+
# --- Install CUDA using Conda
148+
$cudaVersion = '${{ matrix.cuda }}'
149+
$cudaVersionPytorch = '${{ matrix.cuda }}'.Remove('${{ matrix.cuda }}'.LastIndexOf('.')).Replace('.','')
150+
151+
$env:MAMBA_NO_LOW_SPEED_LIMIT = 1
152+
mamba install -y -c nvidia/label/cuda-$cudaVersion cuda-toolkit cuda-runtime
153+
154+
if (!(mamba list cuda)[-1].contains('cuda')) {sleep -s 10; mamba install -y 'cuda' $cudaVersion}
155+
if (!(mamba list cuda)[-1].contains('cuda')) {throw 'CUDA Toolkit failed to install!'}
156+
157+
$env:CUDA_PATH = $env:CONDA_PREFIX
158+
$env:CUDA_HOME = $env:CONDA_PREFIX
159+
if ($IsLinux) {$env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH}
160+
161+
# --- Install dependencies
162+
163+
python -m ensurepip --upgrade
164+
python -m pip install torch==${{ matrix.torch }} --index-url https://download.pytorch.org/whl/cu$cudaVersionPytorch
165+
python -m pip install --upgrade setuptools==69.5.1 build wheel safetensors sentencepiece ninja
166+
167+
# --- Build wheel
168+
169+
$BUILDTAG = "+cu$cudaVersionPytorch-torch${{ matrix.torch }}"
170+
$env:TORCH_CUDA_ARCH_LIST = '${{ matrix.cudaarch }}'
171+
python -m build -n --wheel -C--build-option=egg_info "-C--build-option=--tag-build=$BUILDTAG"
172+
173+
# Build sdist
174+
175+
- name: Build sdist
176+
if: matrix.cuda == '' && matrix.rocm == ''
177+
run: |
178+
# --- Spawn the VS shell
179+
if ($IsWindows) {
180+
Import-Module 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
181+
Enter-VsDevShell -VsInstallPath 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools' -DevCmdArguments '-arch=x64 -host_arch=x64'
182+
$env:DISTUTILS_USE_SDK=1
183+
}
184+
185+
# --- Install dependencies
186+
187+
python -m pip install torch==${{ matrix.torch }} --index-url https://download.pytorch.org/whl/cpu
188+
python -m pip install build wheel ninja
189+
190+
# --- Build wheel
191+
192+
$env:EXLLAMA_NOCOMPILE=1
193+
python -m build -n
194+
195+
# Upload files
196+
197+
- uses: actions/upload-artifact@v4
198+
if: matrix.artname == 'wheel'
199+
with:
200+
name: wheel-${{ matrix.os }}-py${{ matrix.pyver }}-cuda${{ matrix.cuda }}-torch${{ matrix.torch }}
201+
path: ./dist/*
202+
203+
- uses: actions/upload-artifact@v4
204+
if: matrix.artname == 'sdist'
205+
with:
206+
name: 'sdist'
207+
path: ./dist/*
208+
209+
- name: Upload files to GitHub release
210+
if: steps.package_version.outputs.PACKAGE_VERSION != 'None' && inputs.release == '1'
211+
uses: svenstaro/upload-release-action@2.6.1
212+
with:
213+
file: ./dist/*.whl
214+
tag: ${{ format('v{0}', steps.package_version.outputs.PACKAGE_VERSION) }}
215+
file_glob: true
216+
overwrite: true
217+
release_name: ${{ steps.package_version.outputs.PACKAGE_VERSION }}

‎examples/multimodal_grounding_qwen.py

+73-7
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class Model:
5151
current_image: Image or None = None
5252
current_description: str
5353

54-
def __init__(self, model_directory):
54+
def __init__(self, model_directory, bbox_mode: str):
5555
self.model_directory = model_directory
5656
self.config = None
5757
self.vision_model = None
@@ -61,17 +61,22 @@ def __init__(self, model_directory):
6161
self.current_image = None
6262
self.current_emb = None
6363
self.current_description = ""
64+
bbox_funcs = {
65+
"qwen2": self.get_grounding_bb_qwen2,
66+
"qwen25": self.get_grounding_bb_qwen25,
67+
}
68+
self.bbox_func = bbox_funcs[bbox_mode]
6469

6570
def load(self):
6671
"""Load and initialize the things"""
6772
self.config = ExLlamaV2Config(self.model_directory)
68-
self.config.max_seq_len = 16384
73+
self.config.max_seq_len = 8192
6974

7075
self.vision_model = ExLlamaV2VisionTower(self.config)
7176
self.vision_model.load(progress = True)
7277

7378
self.model = ExLlamaV2(self.config)
74-
self.cache = ExLlamaV2Cache(self.model, lazy = True, max_seq_len = 16384)
79+
self.cache = ExLlamaV2Cache(self.model, lazy = True, max_seq_len = 32768)
7580
self.model.load_autosplit(self.cache, progress = True)
7681
self.tokenizer = ExLlamaV2Tokenizer(self.config)
7782

@@ -148,14 +153,21 @@ def inference(self, settext_fn, update_fn):
148153
lastupdate = time.time()
149154
settext_fn(text)
150155
update_fn()
156+
#
157+
# text = \
158+
# """And you may find yourself living in a shotgun shack
159+
# And you may find yourself in another part of the world
160+
# And you may find yourself behind the wheel of a large automobile
161+
# And you may find yourself in a beautiful house, with a beautiful wife
162+
# And you may ask yourself, "Well, how did I get here?\""""
151163

152164
settext_fn(text)
153165
update_fn()
154166
self.current_description = text
155167
print("Image description from model:")
156168
print(text)
157169

158-
def get_grounding_bb(self, start, end) -> tuple:
170+
def get_grounding_bb_qwen2(self, start, end) -> tuple:
159171
"""
160172
Prompt the model again and try to extraxt the bounding box of the image details indicated by selected portion
161173
of the description. We do this by repeating the exact same prompt up to and including the selected text, but
@@ -209,6 +221,55 @@ def get_grounding_bb(self, start, end) -> tuple:
209221

210222
return a, b
211223

224+
def get_grounding_bb_qwen25(self, start, end) -> tuple:
225+
"""
226+
Qwen2.5 works the same way, except the coordinates are no longer normalized and the format is:
227+
"(x0,y0,x1,y1)"
228+
"""
229+
230+
if start >= end:
231+
return None, None
232+
233+
# Including leading space
234+
if start > 0 and self.current_description[start - 1] == " ":
235+
start -= 1
236+
237+
# Repeat the same prompt up to the selection, with grounding tokens added
238+
prompt = self.get_prompt()
239+
prompt += self.current_description[:start]
240+
prompt += "<|object_ref_start|>"
241+
prompt += self.current_description[start:end]
242+
prompt += "<|object_ref_end|><|box_start|>("
243+
244+
bb_string, res = self.generator.generate(
245+
prompt = prompt,
246+
add_bos = True,
247+
max_new_tokens = 28,
248+
stop_conditions = [self.tokenizer.single_id("<|box_end|>")],
249+
gen_settings = ExLlamaV2Sampler.Settings.greedy(),
250+
embeddings = [self.current_emb],
251+
completion_only = True,
252+
return_last_results = True, # debug purposes
253+
)
254+
bb_string = "(" + bb_string
255+
256+
print(f"Generation: {bb_string}")
257+
pprint.pprint(res, indent = 4)
258+
259+
# BB string is in the format "(x0,y0,x1,y1)" with integer coordinates
260+
261+
s = self.current_image.size
262+
try:
263+
d = tuple(map(int, bb_string.strip("()").split(",")))
264+
a = (d[0] / s[0], d[1] / s[1])
265+
b = (d[2] / s[0], d[3] / s[1])
266+
except:
267+
print("No bounding box could be determined")
268+
a, b = None, None
269+
270+
return a, b
271+
272+
212273

213274
class GroundingDemo(QMainWindow):
214275

@@ -472,7 +533,7 @@ def on_selection_made(self, pos):
472533

473534
print(f"Selected span: {start}, {end}")
474535
print(f"Selected text: {repr(self.model.current_description[start:end])}")
475-
a, b = self.model.get_grounding_bb(start, end)
536+
a, b = self.model.bbox_func(start, end)
476537
self.image_label.set_bounding_box(a, b)
477538

478539

@@ -481,9 +542,14 @@ def on_selection_made(self, pos):
481542
# https://huggingface.co/turboderp/Qwen2-VL-7B-Instruct-exl2
482543

483544
def main():
484-
model_dir = "/mnt/str/models/qwen2-vl-7b-instruct-exl2/6.0bpw"
545+
546+
# model_dir = "/mnt/str/models/qwen2-vl-7b-instruct-exl2/6.0bpw"
547+
# bbox_mode = "qwen25"
548+
model_dir = "/mnt/str/models/qwen2.5-vl-7b-instruct-exl2/6.0bpw"
549+
bbox_mode = "qwen25"
550+
485551
app = QApplication(sys.argv)
486-
model = Model(model_dir)
552+
model = Model(model_dir, bbox_mode)
487553
model.load()
488554
window = GroundingDemo(model, model_dir)
489555
window.show()

0 commit comments

Comments
 (0)