switich to last 128 words

main.py

@@ -16,7 +16,7 @@ token_size_limit = None
 # model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-400M-distill")
 tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-1B-distill")
 model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-1B-distill")
-# token_size_limit = 128
+token_size_limit = 128
 
 # T5 model can use "any" sequence lenghth, but memory usage is O(L^2).
 # tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-small")
@@ -25,7 +25,7 @@ model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-1B-distill")
 # model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-base")
 # tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-large")
 # model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-large")
-token_size_limit = 512
+# token_size_limit = 512
 
 # Too large for 16GB
 # tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-xl")