seanp03 commited on
Commit
8856c8f
·
verified ·
1 Parent(s): 1d177ae

added small100 language chunking

Browse files
Files changed (1) hide show
  1. chunking.py +3 -0
chunking.py CHANGED
@@ -23,6 +23,9 @@ def get_max_word_length(target_languages: list[str]) -> int:
23
  "tr": 60,
24
  "sl": 150,
25
  "no": 50,
 
 
 
26
  }
27
 
28
  max_word_length = 700 # Default Chunk Size
 
23
  "tr": 60,
24
  "sl": 150,
25
  "no": 50,
26
+ "lb": 140,
27
+ "sq": 140,
28
+ "sl": 140,
29
  }
30
 
31
  max_word_length = 700 # Default Chunk Size