Spaces:
Paused
Paused
Update train.py
Browse files
train.py
CHANGED
@@ -295,7 +295,7 @@ def main(push_to_hub=True, is_inst_finetune=config.INSTRUCT_FINETUNE_BOOL):
|
|
295 |
# num_new_tokens = update_tokenizer(tokenizer, dataset)
|
296 |
# print(f"Added {num_new_tokens} new tokens to the vocabulary")
|
297 |
|
298 |
-
if INIT == 0:
|
299 |
print("Adding Special Tokens..")
|
300 |
configure_tokenizer(tokenizer)
|
301 |
print("Added Tokens.")
|
|
|
295 |
# num_new_tokens = update_tokenizer(tokenizer, dataset)
|
296 |
# print(f"Added {num_new_tokens} new tokens to the vocabulary")
|
297 |
|
298 |
+
if config.INIT == 0:
|
299 |
print("Adding Special Tokens..")
|
300 |
configure_tokenizer(tokenizer)
|
301 |
print("Added Tokens.")
|