marklysze / llamaindex-rag-linux-cuda Goto Github PK

conda env create -f environment.yml
conda activate LlamaIndexRAGLinux

pip install pip install  notebook ipykernel
jupyter-notebook --allow-root --no-browser --ip=0.0.0.0 --port=27000 --NotebookApp.token='1a4c347ef...........'

import torch

from llama_index.llms import LlamaCPP
from llama_index.llms.llama_utils import messages_to_prompt, completion_to_prompt
llm = LlamaCPP(
    model_url=None, # We'll load locally.
    model_path='./Models/mistral-7b-instruct-v0.1.Q6_K.gguf', # 6-bit model
    temperature=0.1,
    max_new_tokens=1024, # Increasing to support longer responses
    context_window=8192, # Mistral7B has an 8K context-window
    generate_kwargs={},
    # set to at least 1 to use GPU
    model_kwargs={"n_gpu_layers": 33}, # 33 was all that was needed for this and that fit with the RTX 3090's 24GB of memory
    messages_to_prompt=messages_to_prompt,
    completion_to_prompt=completion_to_prompt,
    verbose=True
)

ImportError                               Traceback (most recent call last)
Cell In[3], line 1
----> 1 import torch
      3 from llama_index.llms import LlamaCPP
      4 from llama_index.llms.llama_utils import messages_to_prompt, completion_to_prompt

File ~/anaconda3/envs/LlamaIndexRAGLinux/lib/python3.12/site-packages/torch/__init__.py:235
    233     if USE_GLOBAL_DEPS:
    234         _load_global_deps()
--> 235     from torch._C import *  # noqa: F403
    237 # Appease the type checker; ordinarily this binding is inserted by the
    238 # torch._C module initialization code in C
    239 if TYPE_CHECKING:

ImportError: /home/devteam/anaconda3/envs/LlamaIndexRAGLinux/lib/python3.12/site-packages/torch/lib/../../../../libcusparse.so.12: symbol __nvJitLinkGetLinkedCubin_12_2 version libnvJitLink.so.12 not defined in file libnvJitLink.so.12 with link time reference