Update app.py

app.py

@@ -1,15 +1,15 @@
 import gradio as gr
-from ctransformers import AutoModelForCausalLM
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
 
-# Path to your downloaded GGUF model (change this if needed)
-MODEL_PATH = "qwen2.5-0.5b-instruct-q2_k.gguf"
+# Model name
+MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct-GGUF"
 
-# Load model using ctransformers
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_PATH, model_type="qwen2", gpu_layers=30  # Adjust `gpu_layers` for performance
-)
+# Load tokenizer and model
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.float16, device_map="auto")
 
-def respond(message, history, system_message, max_tokens, temperature, top_p):
+def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p):
     messages = [{"role": "system", "content": system_message}]
     
     # Add chat history to messages
@@ -21,9 +21,16 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
     
     messages.append({"role": "user", "content": message})
     
+    # Tokenize input
+    inputs = tokenizer(message, return_tensors="pt").to("cpu")
+    
     # Generate response
-    response = model(messages, max_new_tokens=max_tokens, temperature=temperature, top_p=top_p)
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs, max_length=max_tokens, temperature=temperature, top_p=top_p
+        )
     
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
     return response
 
 # Define Gradio interface