added small100 language chunking
Browse files- chunking.py +3 -0
chunking.py
CHANGED
@@ -23,6 +23,9 @@ def get_max_word_length(target_languages: list[str]) -> int:
|
|
23 |
"tr": 60,
|
24 |
"sl": 150,
|
25 |
"no": 50,
|
|
|
|
|
|
|
26 |
}
|
27 |
|
28 |
max_word_length = 700 # Default Chunk Size
|
|
|
23 |
"tr": 60,
|
24 |
"sl": 150,
|
25 |
"no": 50,
|
26 |
+
"lb": 140,
|
27 |
+
"sq": 140,
|
28 |
+
"sl": 140,
|
29 |
}
|
30 |
|
31 |
max_word_length = 700 # Default Chunk Size
|