Update app.py

app.py

@@ -74,7 +74,7 @@ def calculate_cache_size(cache):
     return total_memory /(1024*1024)
     
 @st.cache_resource
-def load_quantized_model_and_tokenizer():
+def load_model_and_tokenizer(doc_text_count):
     model_name = "mistralai/Mistral-7B-Instruct-v0.1"  # Configure quantization for 4-bit loading
     quantization_config = BitsAndBytesConfig(
         load_in_4bit=True,  # Enable 4-bit quantization
@@ -87,13 +87,13 @@ def load_quantized_model_and_tokenizer():
         model_name,
         device_map="auto",  # Automatically allocate model to devices
         quantization_config=quantization_config,
-        token=hf_token,
+        model_max_length=1.3*round(doc_text_count * 0.3 + 1)
     )
 
     # Load the tokenizer
     tokenizer = AutoTokenizer.from_pretrained(
         model_name,
-        token=hf_token,
+        
     )
     return tokenizer, model