Update app.py

app.py

@@ -116,18 +116,21 @@ async def tts(ws: WebSocket):
         offset_len = ids.size(1)          # wie viele Tokens existieren schon
         last_tok   = None
         buf        = []
-        masker.buffer_pos = 0 # Initialize buffer position for masker
+        # masker.buffer_pos = 0 # Removed initialization here
 
         while True:
-            # --- Mini‑Generate (Cache Disabled for Debugging) -------------------------------------------
+            # Update buffer_pos based on current buffer length before generation
+            masker.buffer_pos = len(buf)
+
+            # --- Mini‑Generate (Cache Re-enabled) -------------------------------------------
             gen = model.generate(
-                input_ids      = ids, # Always use full sequence
-                attention_mask = attn, # Always use full attention mask
-                # past_key_values= past, # Disabled cache
+                input_ids       = ids if past is None else torch.tensor([[last_tok]], device=device), # Reverted
+                attention_mask  = attn if past is None else None, # Reverted
+                past_key_values = past, # Re-enabled cache
                 max_new_tokens = CHUNK_TOKENS,
                 logits_processor=[masker],
                 do_sample=True, temperature=0.7, top_p=0.95,
-                use_cache=False, # Disabled cache
+                use_cache=True, # Re-enabled cache
                 return_dict_in_generate=True,
                 return_legacy_cache=True
             )
@@ -139,10 +142,10 @@ async def tts(ws: WebSocket):
                 break
             offset_len += len(new)
 
-            # ----- Update ids and attn with the full sequence (Cache Disabled) ---------
-            ids = torch.tensor([seq], device=device)
-            attn = torch.ones_like(ids)
-            # past = gen.past_key_values # Disabled cache access
+            # ----- Update past and last_tok (Cache Re-enabled) ---------
+            # ids = torch.tensor([seq], device=device) # Removed
+            # attn = torch.ones_like(ids) # Removed
+            past = gen.past_key_values # Re-enabled cache access
             last_tok = new[-1]
 
             print("new tokens:", new[:25], flush=True)
@@ -155,12 +158,12 @@ async def tts(ws: WebSocket):
                     buf.clear()
                     continue
                 buf.append(t - AUDIO_BASE)
-                masker.buffer_pos += 1 # Increment buffer position
+                # masker.buffer_pos += 1 # Removed increment here
                 if len(buf) == 7:
                     await ws.send_bytes(decode_block(buf))
                     buf.clear()
                     masker.sent_blocks = 1      # ab jetzt EOS zulässig
-                    masker.buffer_pos = 0 # Reset buffer position after sending a block
+                    # masker.buffer_pos = 0 # Removed reset here
 
     except (StopIteration, WebSocketDisconnect):
         pass