"CUDA error: the provided PTX was compiled with an unsupported toolchain." with 0.3.0

[phiren961]

I've got this error with this code (version 0.3):

conda remove -n rewrite2 --all -y
conda create -n rewrite2 python=3.11 -y
conda activate rewrite2
export CUDA_HOME=/usr/local/cuda-12.3
export PATH=$CUDA_HOME/bin:$PATH
export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
sudo ln -sfn /usr/local/cuda-12.3 /usr/local/cuda
export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH
CMAKE_ARGS="-DGGML_CUDA=on" FORCE_CMAKE=1 pip install --force-reinstall --no-cache-dir llama-cpp-python

but it's functioning well with this code (version 0.2.9):

conda remove -n rewrite2 --all -y
conda create -n rewrite2 python=3.11 -y
conda activate rewrite2
export CUDA_HOME=/usr/local/cuda-12.3
export PATH=$CUDA_HOME/bin:$PATH
export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
sudo ln -sfn /usr/local/cuda-12.3 /usr/local/cuda
export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH
wget https://github.com/abetlen/llama-cpp-python/releases/download/v0.2.90-cu123/llama_cpp_python-0.2.90-cp311-cp311-linux_x86_64.whl
CMAKE_ARGS="-DGGML_CUDA=on" FORCE_CMAKE=1 pip install --force-reinstall llama_cpp_python-0.2.90-cp311-cp311-linux_x86_64.whl

[phiren961]

in both case here is the script :

#!/bin/bash
export CUDA_HOME=/usr/local/cuda-12.3
export PATH=$CUDA_HOME/bin:$PATH
export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
python3.11 << END

def main():
    askLLMccp("what's your name ?", "/media/az/nvme1-2To/LLM MODELS/Qwen2.5-32B-Instruct-Q4_K_M.gguf")

def askLLMccp(prompt, modelPath):
    llm = Llama(
        model_path=modelPath,
        n_gpu_layers=-1, 
        # seed=1337, 
        # n_ctx=2048, 
    )
    output = llm(
        prompt, # Prompt
        max_tokens=512, 
        stop=["Q:", "\n"], 
        echo=False
    )
    response_text = output['choices'][0]['text']
    print(response_text)

from llama_cpp import Llama

if __name__ == "__main__":
	main()
	exit()
END

[phiren961]

I found the solution :

export CUDA_HOME=/usr/local/cuda-12.3
export PATH=$CUDA_HOME/bin:$PATH
export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
sudo ln -sfn /usr/local/cuda-12.3 /usr/local/cuda
export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH
CUDACXX=/usr/local/cuda-12.3/bin/nvcc CMAKE_ARGS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all-major" FORCE_CMAKE=1 pip install llama-cpp-python --no-cache-dir --force-reinstall --upgrade