Added history

run.py

@@ -2,7 +2,7 @@
 # Title:  Gradio Interface to LLM-chatbot (for recommending AI) with RAG-funcionality and ChromaDB on HF-Hub 
 # Author: Andreas Fischer
 # Date:   December 30th, 2023
-# Last update: February 27th, 2024
+# Last update: May 27th, 2024
 ##############################################################################################################
 
 
@@ -57,14 +57,53 @@ print(collection.count())
 
 # Model
 #-------
+onPrem=False
+myModel="mistralai/Mixtral-8x7B-Instruct-v0.1" 
+if(onPrem==False): 
+  modelPath=myModel
+  from huggingface_hub import InferenceClient
+  import gradio as gr
+  client = InferenceClient(
+    model=modelPath,
+    #token="hf_..."
+  )
+else:
+  import os
+  import requests
+  import subprocess
+  #modelPath="/home/af/gguf/models/c4ai-command-r-v01-Q4_0.gguf"
+  #modelPath="/home/af/gguf/models/Discolm_german_7b_v1.Q4_0.gguf"
+  modelPath="/home/af/gguf/models/Mixtral-8x7b-instruct-v0.1.Q4_0.gguf"
+  if(os.path.exists(modelPath)==False):
+    #url="https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-GGUF/resolve/main/discolm_german_7b_v1.Q4_0.gguf?download=true"
+    url="https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q4_0.gguf?download=true"
+    response = requests.get(url)
+    with open("./Mixtral-8x7b-instruct.gguf", mode="wb") as file:
+      file.write(response.content)
+    print("Model downloaded")  
+    modelPath="./Mixtral-8x7b-instruct.gguf"
+  print(modelPath)
+  n="20" 
+  if("Mixtral-8x7b-instruct" in modelPath): n="0" # mixtral seems to cause problems here...
+  command = ["python3", "-m", "llama_cpp.server", "--model", modelPath, "--host", "0.0.0.0", "--port", "2600", "--n_threads", "8", "--n_gpu_layers", n]
+  subprocess.Popen(command)
+  print("Server ready!")
 
-from huggingface_hub import InferenceClient
-import gradio as gr
-modelPath="mistralai/Mixtral-8x7B-Instruct-v0.1"
-client = InferenceClient(
-    modelPath
-    #"mistralai/Mistral-7B-Instruct-v0.1"
-)
+
+# Check template
+#----------------
+if(False):
+  from transformers import AutoTokenizer
+  #mod="mistralai/Mixtral-8x22B-Instruct-v0.1"
+  #mod="mistralai/Mixtral-8x7b-instruct-v0.1"
+  mod="VAGOsolutions/Llama-3-SauerkrautLM-8b-Instruct"
+  tok=AutoTokenizer.from_pretrained(mod) #,token="hf_...")
+  cha=[{"role":"system","content":"A"},{"role":"user","content":"B"},{"role":"assistant","content":"C"}]
+  res=tok.apply_chat_template(cha)
+  print(tok.decode(res))
+  cha=[{"role":"user","content":"U1"},{"role":"assistant","content":"A1"},{"role":"user","content":"U2"},{"role":"assistant","content":"A2"}]
+  res=tok.apply_chat_template(cha)
+  print(tok.decode(res))
 
 
 # Gradio-GUI
@@ -72,19 +111,30 @@ client = InferenceClient(
 
 import gradio as gr
 import json
+import re
 
-def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=None, zeichenlimit=None,historylimit=4, removeHTML=False): 
+def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=None, zeichenlimit=None,historylimit=4, removeHTML=True): 
   startOfString=""
   if zeichenlimit is None: zeichenlimit=1000000000 # :-)
   template0=" [INST]{system}\n  [/INST] </s>" 
   template1=" [INST] {message} [/INST]"
   template2=" {response}</s>"
+  if("command-r" in modelPath): #https://huggingface.co/CohereForAI/c4ai-command-r-v01
+    ## <BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
+    template0="<BOS_TOKEN><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|> {system}<|END_OF_TURN_TOKEN|>" 
+    template1="<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{message}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
+    template2="{response}<|END_OF_TURN_TOKEN|>"
   if("Gemma-" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
     template0="<start_of_turn>user{system}</end_of_turn>" 
     template1="<start_of_turn>user{message}</end_of_turn><start_of_turn>model"
-    template2="{response}</end_of_turn>"
-  if("Mixtral-8x7b-instruct" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
+    template2="{response}</end_of_turn>"      
+  if("Mixtral-8x22B-Instruct" in modelPath): # AutoTokenizer: <s>[INST] U1[/INST] A1</s>[INST] U2[/INST] A2</s>
     startOfString="<s>"
+    template0="[INST]{system}\n  [/INST] </s>"  
+    template1="[INST] {message}[/INST]"
+    template2=" {response}</s>"
+  if("Mixtral-8x7b-instruct" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
+    startOfString="<s>"                     # AutoTokenzizer: <s> [INST] U1 [/INST]A1</s> [INST] U2 [/INST]A2</s>
     template0=" [INST]{system}\n  [/INST] </s>"  
     template1=" [INST] {message} [/INST]"
     template2=" {response}</s>"
@@ -100,7 +150,11 @@ def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=
   if(("Discolm_german_7b" in modelPath) or ("SauerkrautLM-7b-HerO" in modelPath)):  #https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-HerO
     template0="<|im_start|>system\n{system}<|im_end|>\n"
     template1="<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
-    template2="{response}<|im_end|>\n"
+    template2="{response}<|im_end|>\n"    
+  if("Llama-3-SauerkrautLM-8b-Instruct" in modelPath):  #https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-HerO
+    template0="<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system}<|eot_id|>"
+    template1="<|start_header_id|>user<|end_header_id|>\n\n{message}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+    template2="{response}<|eot_id|>\n"        
   if("WizardLM-13B-V1.2" in modelPath): #https://huggingface.co/WizardLM/WizardLM-13B-V1.2
     template0="{system} " #<s>
     template1="USER: {message} ASSISTANT: "
@@ -128,8 +182,9 @@ def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=
   return startOfString+prompt
 
 
+
 def response(
-    prompt, history, temperature=0.9, max_new_tokens=500, top_p=0.95, repetition_penalty=1.0,
+    message, history, temperature=0.9, max_new_tokens=500, top_p=0.95, repetition_penalty=1.0,
 ):
     temperature = float(temperature)
     if temperature < 1e-2: temperature = 1e-2
@@ -144,7 +199,7 @@ def response(
     )
     addon=""
     results=collection.query(
-      query_texts=[prompt],
+      query_texts=[message],
       n_results=2,
       #where={"source": "google-docs"}
       #where_document={"$contains":"search_string"}
@@ -157,9 +212,18 @@ def response(
     print(combination)
     if(len(results)>1):
       addon=" Bitte berücksichtige bei deiner Antwort ggf. folgende Auszüge aus unserer Datenbank, sofern sie für die Antwort erforderlich sind. Beantworte die Frage knapp und präzise. Ignoriere unpassende Datenbank-Auszüge OHNE sie zu kommentieren, zu erwähnen oder aufzulisten:\n"+"\n".join(results)
-    system="Du bist ein deutschsprachiges KI-basiertes Assistenzsystem, das zu jedem Anliegen möglichst geeignete KI-Tools empfiehlt."+addon+"\n\nUser-Anliegen:"   
+    system="Du bist ein deutschsprachiges KI-basiertes Assistenzsystem, das zu jedem Anliegen möglichst geeignete KI-Tools empfiehlt." #+addon #+"\n\nUser-Anliegen:"   
     #body={"prompt":system+"### Instruktion:\n"+message+"\n\n### Antwort:","max_tokens":500, "echo":"False","stream":"True"} #e.g. SauerkrautLM
-    formatted_prompt = extend_prompt(system+"\n"+prompt,  None) #history)
+    #formatted_prompt = extend_prompt(system+"\n"+prompt,  None) #history)
+    prompt=extend_prompt(
+      message,                  # current message of the user
+      history,                  # complete history 
+      system,                   # system prompt
+      addon,                    # RAG-component added to the system prompt
+      None,                     # fictive first words of the AI (neither displayed nor stored)
+      historylimit=4,           # number of past messages to consider for response to current message
+      removeHTML=True           # remove HTML-components from History (to prevent bugs with Markdown)
+    )
     stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
     output = ""
     for response in stream: