fix slow response

app.py

@@ -1,10 +1,12 @@
 import os
+import time
 import warnings
 warnings.filterwarnings("ignore", category=UserWarning, module="torch._utils")
 
 from transformers import GPT2LMHeadModel, GPT2Tokenizer
 import torch
 import gradio as gr
+import psutil
 
 # Load model and tokenizer
 model_id = "NlpHUST/gpt2-vietnamese"
@@ -29,8 +31,18 @@ model.eval()
 print(f"Device: {device}")
 print(f"Memory allocated: {torch.cuda.memory_allocated(device)/1e9:.2f} GB" if torch.cuda.is_available() else "CPU only")
 
-def generate_text(prompt, max_length=100, temperature=1.0):
+def print_system_resources():
+    cpu_percent = psutil.cpu_percent(interval=1)
+    memory = psutil.virtual_memory()
+    print(f"CPU usage: {cpu_percent}%")
+    print(f"Memory usage: {memory.percent}% ({memory.used/1e9:.2f}/{memory.total/1e9:.2f} GB)")
+
+# Call before generation
+print_system_resources()
+
+def generate_text(prompt, max_length=50, temperature=1.0):
     try:
+        start_time = time.time()
         # Encode input with attention mask
         inputs = tokenizer(
             prompt,
@@ -44,13 +56,18 @@ def generate_text(prompt, max_length=100, temperature=1.0):
         outputs = model.generate(
             input_ids=inputs["input_ids"],
             attention_mask=inputs["attention_mask"],
-            max_length=max_length,
+            max_new_tokens=30,  # Limit new tokens to reduce computation
             temperature=temperature,
             do_sample=True,
-            num_beams=1,
-            pad_token_id=tokenizer.pad_token_id
+            num_beams=3,  # Use beam search for faster generation
+            no_repeat_ngram_size=2,  # Prevent repetitive phrases
+            pad_token_id=tokenizer.pad_token_id,
+            early_stopping=True  # Stop when generation is complete
         )
-        return tokenizer.decode(outputs[0], skip_special_tokens=True)
+        generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        elapsed_time = time.time() - start_time
+        print(f"Generation time: {elapsed_time:.2f} seconds")
+        return generated_text
     except Exception as e:
         return f"Error generating text: {e}"
 
@@ -59,7 +76,7 @@ demo = gr.Interface(
     fn=generate_text,
     inputs=[
         gr.Textbox(label="Nhập văn bản đầu vào", placeholder="Viết gì đó bằng tiếng Việt..."),
-        gr.Slider(20, 300, value=100, step=10, label="Độ dài tối đa"),
+        gr.Slider(20, 100, value=50, step=10, label="Độ dài tối đa"),
         gr.Slider(0.5, 1.5, value=1.0, step=0.1, label="Nhiệt độ (Temperature)")
     ],
     outputs="text",