Upload 2 files

Using a quantised version of the gemma-9b (neuralmagic/gemma-2-9b-it-quantized.w4a16)

app.py

@@ -161,8 +161,8 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 # #model = AutoModelForCausalLM.from_pretrained(READER_MODEL_NAME)
 # model = AutoModelForCausalLM.from_pretrained(READER_MODEL_NAME).to(device)
 
-#model_id = "neuralmagic/gemma-2-9b-it-quantized.w4a16"
-model_id = "TheBloke/Gemma-2-7B-IT-GGUF"
+model_id = "neuralmagic/gemma-2-9b-it-quantized.w4a16"
+#model_id = "TheBloke/Gemma-2-7B-IT-GGUF"
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,

requirements.txt

@@ -13,4 +13,6 @@ openai-whisper
 uuid
 textwrap3
 python-dotenv
+pyngrok
+compressed-tensors