Update app.py

app.py

@@ -14,9 +14,9 @@ flores =  load_dataset("facebook/flores", "eng_Latn-ukr_Cyrl")
 dataset = flores[subset]
 
 fig, (axl, axr) = plt.subplots(1, 2, figsize=(10,5))
-axl.hist(dataset.map(lambda x: {'num_tokens':len(tok(x['sentence_eng_Latn'])['input_ids'])})['num_tokens'])
+axl.hist(dataset.map(lambda x: {'num_tokens':len(tokenizer(x['sentence_eng_Latn'])['input_ids'])})['num_tokens'])
 axl.set_title('eng mistral tokens')
-axr.hist(dataset.map(lambda x: {'num_tokens':len(tok(x['sentence_ukr_Cyrl'])['input_ids'])})['num_tokens'])
+axr.hist(dataset.map(lambda x: {'num_tokens':len(tokenizer(x['sentence_ukr_Cyrl'])['input_ids'])})['num_tokens'])
 axr.set_title('ukr mistral tokens')
 
 st.dataframe(pd.DataFrame(dataset[subset]))