Update run.py

run.py

@@ -119,17 +119,17 @@ else:
   #modelPath="/home/af/gguf/models/nous-hermes-2-mixtral-8x7b-dpo.Q4_0.gguf"
   if(os.path.exists(modelPath)==False):
     #url="https://huggingface.co/TheBloke/WizardLM-13B-V1.2-GGUF/resolve/main/wizardlm-13b-v1.2.Q4_0.gguf"
-    #url="https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q4_0.gguf?download=true"
+    url="https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q4_0.gguf?download=true"
     #url="https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_0.gguf?download=true"
-    url="https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-GGUF/resolve/main/discolm_german_7b_v1.Q4_0.gguf?download=true"
+    #url="https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-GGUF/resolve/main/discolm_german_7b_v1.Q4_0.gguf?download=true"
     response = requests.get(url)
-    with open("./model.gguf", mode="wb") as file:
+    with open("./Mixtral-8x7b-instruct.gguf", mode="wb") as file:
       file.write(response.content)
     print("Model downloaded")  
-    modelPath="./model.gguf"
+    modelPath="./Mixtral-8x7b-instruct.gguf"
   print(modelPath)
   n="20" 
-  if("mixtral-8x7b-instruct" in modelPath): n="0" # mixtral seems to cause problems here...
+  if("Mixtral-8x7b-instruct" in modelPath): n="0" # mixtral seems to cause problems here...
   command = ["python3", "-m", "llama_cpp.server", "--model", modelPath, "--host", "0.0.0.0", "--port", "2600", "--n_threads", "8", "--n_gpu_layers", n]
   subprocess.Popen(command)
   print("Server ready!")
@@ -143,44 +143,40 @@ else:
 # Gradio-GUI
 #------------
 
-def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=None, zeichenlimit=None,historylimit=4): 
+
+def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=None, zeichenlimit=None,historylimit=4): #float("Inf")
   startOfString=""
   if zeichenlimit is None: zeichenlimit=1000000000 # :-)
   template0=" [INST]{system}\n  [/INST] </s>" 
   template1=" [INST] {message} [/INST]"
   template2=" {response}</s>"
-  if("mixtral-8x7b-instruct" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
+  if("Gemma-" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
+    template0="<start_of_turn>user{system}</end_of_turn>" 
+    template1="<start_of_turn>user{message}</end_of_turn><start_of_turn>model"
+    template2="{response}</end_of_turn>"
+  if("Mixtral-8x7b-instruct" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
     startOfString="<s>"
-    template0=" [INST]{system}\n  [/INST] </s>" 
+    template0=" [INST]{system}\n  [/INST] </s>"  
     template1=" [INST] {message} [/INST]"
     template2=" {response}</s>"
   if("Mistral-7B-Instruct" in modelPath): #https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2
     startOfString="<s>"
-    template0="[INST]{system}\n [/INST]</s>" 
+    template0="[INST]{system}\n [/INST]</s>"
     template1="[INST] {message} [/INST]"
     template2=" {response}</s>"
-  if("discolm_german_7b" in modelPath): #https://huggingface.co/DiscoResearch/DiscoLM_German_7b_v1
-    template0="<|im_start|>system\n{system}<|im_end|>\n"
-    template1="<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
-    template2="{response}<|im_end|>\n"
-  if("SauerkrautLM-7b-HerO" in modelPath):  #https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-HerO
-    template0="<|im_start|>system\n{system}<|im_end|>\n"
-    template1="<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
-    template2="{response}<|im_end|>\n"
-  if("gemma-" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
-    template0="<start_of_turn>user{system}</end_of_turn>" 
-    template1="<start_of_turn>user{message}</end_of_turn><start_of_turn>model"
-    template2="{response}</end_of_turn>"  
-  if("openchat-3.5" in modelPath): #https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF
-    startOfString="<s>"
+  if("Openchat-3.5" in modelPath): #https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF
     template0="GPT4 Correct User: {system}<|end_of_turn|>GPT4 Correct Assistant: Okay.<|end_of_turn|>"
     template1="GPT4 Correct User: {message}<|end_of_turn|>GPT4 Correct Assistant: "
     template2="{response}<|end_of_turn|>"
+  if(("Discolm_german_7b" in modelPath) or ("SauerkrautLM-7b-HerO" in modelPath)):  #https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-HerO
+    template0="<|im_start|>system\n{system}<|im_end|>\n"
+    template1="<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
+    template2="{response}<|im_end|>\n"
   if("WizardLM-13B-V1.2" in modelPath): #https://huggingface.co/WizardLM/WizardLM-13B-V1.2
     template0="{system} " #<s>
     template1="USER: {message} ASSISTANT: "
     template2="{response}</s>"
-  if("phi-2" in modelPath): #https://huggingface.co/TheBloke/phi-2-GGUF
+  if("Phi-2" in modelPath): #https://huggingface.co/TheBloke/phi-2-GGUF
     template0="Instruct: {system}\nOutput: Okay.\n"
     template1="Instruct: {message}\nOutput:"
     template2="{response}\n"  
@@ -188,17 +184,18 @@ def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=
   if RAGAddon is not None:
     system += RAGAddon
   if system is not None:
-    prompt += template0.format(system=system.strip()) 
+    prompt += template0.format(system=system) #"<s>"
   if history is not None:
     for user_message, bot_response in history[-historylimit:]:
-      if user_message is not None: prompt += template1.format(message=user_message[:zeichenlimit].strip())  
-      if bot_response is not None: prompt += template2.format(response=bot_response[:zeichenlimit].strip()) 
-  if message is not None: prompt += template1.format(message=message[:zeichenlimit].strip())                
+      if user_message is not None: prompt += template1.format(message=user_message[:zeichenlimit])  #"[INST] {user_prompt} [/INST] "
+      if bot_response is not None: prompt += template2.format(response=bot_response[:zeichenlimit]) #"{bot_response}</s> "
+  if message is not None: prompt += template1.format(message=message[:zeichenlimit])                #"[INST] {message} [/INST]"
   if system2 is not None:
-    prompt += system2.strip()
+    prompt += system2
   return startOfString+prompt
 
 
+
 import gradio as gr
 import requests
 import json