chore: install flash-attn at runtime

app.py

@@ -19,6 +19,18 @@ except:
             return lambda x: x
 
 
+try:
+    import flash_attn
+except:
+    import subprocess
+
+    subprocess.run(
+        "pip install flash-attn --no-build-isolation",
+        env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
+        shell=True,
+    )
+
+
 MODEL_NAME = "hatakeyama-llm-team/Tanuki-8B-Instruct"
 
 quantization_config = BitsAndBytesConfig(load_in_8bit=True)

requirements.txt

@@ -4,4 +4,3 @@ accelerate==0.30.1
 transformers==4.41.2
 spaces==0.28.3
 bitsandbytes==0.43.1
-flash-attn==2.5.9.post1