Update app.py

app.py

@@ -32,6 +32,40 @@ client = OpenAI(
     base_url="https://api-inference.huggingface.co/v1/",
     api_key=TOKEN,
 )
+
+
+def qwen_api(user_message, top_p=0.9,temperature=0.7, system_message='', max_tokens=1024, gradio_history=[]):
+    history = []
+    if gradio_history:
+        for message in history:
+            if message:
+                history.append({"role": "user", "content": message[0]})
+                history.append({"role": "assistant", "content": message[1]})
+
+    if system_message!='':
+        history.append({'role': 'system', 'content': system_message})
+    history.append({"role": "user", "content": user_message})
+
+    response = ""
+    for message in  client.chat.completions.create(
+        model="meta-llama/Meta-Llama-3-8B-Instruct",
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+        messages=history,
+    ):
+        token = message.choices[0].delta.content
+        response += token
+        yield response
+    return response
+
+llm = ChatOpenAI(
+        model="meta-llama/Meta-Llama-3-8B-Instruct",
+        temperature=0.8,)
+
+
+
 embedding = load_embedding_mode()
 db = Chroma(persist_directory='/VecterStore2_512_txt/VecterStore2_512_txt', embedding_function=embedding)
 prompt_template = """
@@ -97,7 +131,7 @@ def respond(
 chatbot = gr.Chatbot(height=600)
 
 demo = gr.ChatInterface(
-    respond,
+    chat,
     fill_height=True,
     chatbot=chatbot,
     additional_inputs=[