jl3676 commited on
Commit
8c638e2
·
verified ·
1 Parent(s): b95863f

Revert to original tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -2123,5 +2123,5 @@
2123
  ],
2124
  "model_max_length": 131072,
2125
  "pad_token": "<pad>",
2126
- "tokenizer_class": "CustomChatTokenizer"
2127
  }
 
2123
  ],
2124
  "model_max_length": 131072,
2125
  "pad_token": "<pad>",
2126
+ "tokenizer_class": "PreTrainedTokenizerFast"
2127
  }