adjust generation time

app.py

@@ -52,6 +52,7 @@ model.eval()
 model = torch.quantization.quantize_dynamic(
     model, {torch.nn.Linear}, dtype=torch.qint8
 )
+print(f"Model quantized: {model.__class__.__name__}")
 
 # Print device and memory info for debugging
 print(f"Device: {device}")
@@ -62,18 +63,20 @@ def clean_text(text):
     """Clean generated text by removing non-alphabetic characters and incomplete sentences."""
     text = re.sub(r'[^\w\s.,!?àáâãèéêìíòóôõùúýăđĩũơưạảấầẩẫậắằẳẵặẹẻẽếềểễệỉịọỏốồổỗộớờởỡợụủứừửữựỳỵỷỹ]', '', text)
     text = re.sub(r'\s+', ' ', text).strip()
-    # Keep only complete sentences (ending with punctuation)
+    # Keep only complete sentences (ending with .!?)
     sentences = re.split(r'(?<=[.!?])\s+', text)
     complete_sentences = [s for s in sentences if re.search(r'[.!?]$', s)]
     if complete_sentences:
-        text = ' '.join(complete_sentences)
-    else:
-        # Fallback: Keep until last valid word
-        words = text.split()
-        text = ' '.join(words[:-1]) if len(words) > 1 else text
-    return text
+        return ' '.join(complete_sentences)
+    # Fallback: Cut at last comma if present
+    last_comma = text.rfind(',')
+    if last_comma != -1:
+        return text[:last_comma].strip()
+    # Fallback: Cut at last valid word
+    words = text.split()
+    return ' '.join(words[:-1]) if len(words) > 1 else text
 
-def generate_text(prompt, max_length=40, temperature=1.0):
+def generate_text(prompt, max_length=30, temperature=1.0):
     try:
         start_time = time.time()
         print_system_resources()
@@ -91,11 +94,10 @@ def generate_text(prompt, max_length=40, temperature=1.0):
         outputs = model.generate(
             input_ids=inputs["input_ids"],
             attention_mask=inputs["attention_mask"],
-            max_new_tokens=10,  # Reduce for speed
-            min_length=5,
+            max_new_tokens=8,  # Reduce for speed
+            min_length=3,
             do_sample=True,
-            top_k=20,  # Reduce for speed
-            top_p=0.85,  # Prioritize high-quality tokens
+            top_k=15,  # Reduce for speed
             temperature=temperature,
             pad_token_id=tokenizer.pad_token_id
         )
@@ -107,7 +109,6 @@ def generate_text(prompt, max_length=40, temperature=1.0):
         print(f"Generation time: {elapsed_time:.2f} seconds")
         # Clear memory cache
         gc.collect()
-        torch.cuda.empty_cache() if torch.cuda.is_available() else None
         return cleaned_text
     except Exception as e:
         return f"Error generating text: {e}"
@@ -121,7 +122,7 @@ demo = gr.Interface(
             placeholder="Viết gì đó bằng tiếng Việt...",
             value="Hôm nay là một ngày đẹp trời"  # Default text
         ),
-        gr.Slider(20, 100, value=40, step=10, label="Độ dài tối đa"),
+        gr.Slider(20, 100, value=30, step=10, label="Độ dài tối đa"),
         gr.Slider(0.7, 1.0, value=1.0, step=0.1, label="Nhiệt độ (Temperature)")
     ],
     outputs="text",