Update app.py

app.py

@@ -1,15 +1,15 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
+from ctransformers import AutoModelForCausalLM
 
-# Model name
-MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct-GGUF"
+# Path to your downloaded GGUF model (change this if needed)
+MODEL_PATH = "qwen2.5-0.5b-instruct-q2_k.gguf"
 
-# Load tokenizer and model
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.float16, device_map="auto")
+# Load model using ctransformers
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_PATH, model_type="qwen2", gpu_layers=30  # Adjust `gpu_layers` for performance
+)
 
-def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p):
+def respond(message, history, system_message, max_tokens, temperature, top_p):
     messages = [{"role": "system", "content": system_message}]
     
     # Add chat history to messages
@@ -21,16 +21,9 @@ def respond(message, history: list[tuple[str, str]], system_message, max_tokens,
     
     messages.append({"role": "user", "content": message})
     
-    # Tokenize input
-    inputs = tokenizer(message, return_tensors="pt").to("cpu")
-    
     # Generate response
-    with torch.no_grad():
-        outputs = model.generate(
-            **inputs, max_length=max_tokens, temperature=temperature, top_p=top_p
-        )
+    response = model(messages, max_new_tokens=max_tokens, temperature=temperature, top_p=top_p)
     
-    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
     return response
 
 # Define Gradio interface