Update parameters for Indonesian (and default ones)

parameters_filtering.py

@@ -26,34 +26,34 @@ parameters_filtering_default = {
     "cond_check_number_words": True,
     "tokenization": False,
     "strip_characters": special_characters_default,
-    "number_words_min_cutoff": 1,
+    "number_words_min_cutoff": 10,
     "number_words_max_cutoff": 100000,
     "cond_check_character_repetition_removal": True,
     "character_repetition_length": 10,
-    "character_repetition_max_cutoff": 0.106,
+    "character_repetition_max_cutoff": 0.2,
     "cond_check_word_repetition_removal": True,
     "word_repetition_length": 5,
-    "word_repetition_max_cutoff": 0.19,
+    "word_repetition_max_cutoff": 0.3,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.4,
     "cond_words_augmentation": False,
     "words_augmentation_group_sizes": [],
     "words_augmentation_join_char": "",
-    "cond_check_stopwords": False,
-    "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
-    "flagged_words_max_cutoff": 0.2,
+    "cond_check_stopwords": True,
+    "stopwords_min_cutoff": 0.1,
+    "cond_check_flagged_words": True,
+    "flagged_words_max_cutoff": 0.1,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.70,
     "cond_check_perplexity": False,
-    "perplexity_max_cutoff": 3000000,
+    "perplexity_max_cutoff": 10000,
 }
 
 parameters_filtering_af = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 25,
@@ -76,7 +76,7 @@ parameters_filtering_af = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.6,
@@ -87,7 +87,7 @@ parameters_filtering_af = {
 parameters_filtering_ar = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 25,
@@ -110,7 +110,7 @@ parameters_filtering_ar = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -121,7 +121,7 @@ parameters_filtering_ar = {
 parameters_filtering_arz = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 25,
@@ -144,7 +144,7 @@ parameters_filtering_arz = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -155,7 +155,7 @@ parameters_filtering_arz = {
 parameters_filtering_as = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 25,
@@ -178,7 +178,7 @@ parameters_filtering_as = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -189,7 +189,7 @@ parameters_filtering_as = {
 parameters_filtering_bn = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -212,7 +212,7 @@ parameters_filtering_bn = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0.05,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -246,7 +246,7 @@ parameters_filtering_ca = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0.25,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.1,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.8,
@@ -291,7 +291,7 @@ parameters_filtering_en = {
 parameters_filtering_es = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -314,7 +314,7 @@ parameters_filtering_es = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0.2,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -325,7 +325,7 @@ parameters_filtering_es = {
 parameters_filtering_eu = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 35,
@@ -348,7 +348,7 @@ parameters_filtering_eu = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -382,7 +382,7 @@ parameters_filtering_fr = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0.27,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.008,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.8,
@@ -393,7 +393,7 @@ parameters_filtering_fr = {
 parameters_filtering_gu = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -416,7 +416,7 @@ parameters_filtering_gu = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -427,7 +427,7 @@ parameters_filtering_gu = {
 parameters_filtering_hi = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 25,
@@ -450,7 +450,7 @@ parameters_filtering_hi = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -461,41 +461,41 @@ parameters_filtering_hi = {
 parameters_filtering_id = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
     "cond_check_number_words": True,
     "tokenization": False,
     "strip_characters": special_characters_default,
-    "number_words_min_cutoff": 1,
+    "number_words_min_cutoff": 15,
     "number_words_max_cutoff": 100000,
     "cond_check_character_repetition_removal": True,
     "character_repetition_length": 10,
-    "character_repetition_max_cutoff": 0.106,
+    "character_repetition_max_cutoff": 0.15,
     "cond_check_word_repetition_removal": True,
     "word_repetition_length": 5,
-    "word_repetition_max_cutoff": 0.19,
+    "word_repetition_max_cutoff": 0.20,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
-    "special_characters_max_cutoff": 0.25,
+    "special_characters_max_cutoff": 0.34,
     "cond_words_augmentation": False,
     "words_augmentation_group_sizes": [],
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
-    "stopwords_min_cutoff": 0.25,
-    "cond_check_flagged_words": False,
-    "flagged_words_max_cutoff": 0.2,
+    "stopwords_min_cutoff": 0.15,
+    "cond_check_flagged_words": True,
+    "flagged_words_max_cutoff": 0.01,
     "cond_check_lang_id": True,
-    "lang_id_min_cutoff": 0.75,
+    "lang_id_min_cutoff": 0.7,
     "cond_check_perplexity": True,
-    "perplexity_max_cutoff": 2500000,
+    "perplexity_max_cutoff": 5000,
 }
 
 parameters_filtering_kn = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 50,
@@ -518,7 +518,7 @@ parameters_filtering_kn = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -529,7 +529,7 @@ parameters_filtering_kn = {
 parameters_filtering_ml = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 50,
@@ -552,7 +552,7 @@ parameters_filtering_ml = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -563,7 +563,7 @@ parameters_filtering_ml = {
 parameters_filtering_mr = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -586,7 +586,7 @@ parameters_filtering_mr = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -620,7 +620,7 @@ parameters_filtering_pt = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0.2,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.007,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.6,
@@ -631,7 +631,7 @@ parameters_filtering_pt = {
 parameters_filtering_sw = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -654,7 +654,7 @@ parameters_filtering_sw = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -665,7 +665,7 @@ parameters_filtering_sw = {
 parameters_filtering_ta = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 50,
@@ -688,7 +688,7 @@ parameters_filtering_ta = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -699,7 +699,7 @@ parameters_filtering_ta = {
 parameters_filtering_te = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 35,
@@ -722,7 +722,7 @@ parameters_filtering_te = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -733,7 +733,7 @@ parameters_filtering_te = {
 parameters_filtering_ur = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -756,7 +756,7 @@ parameters_filtering_ur = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -767,7 +767,7 @@ parameters_filtering_ur = {
 parameters_filtering_vi = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -790,7 +790,7 @@ parameters_filtering_vi = {
     "words_augmentation_join_char": " ",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -801,7 +801,7 @@ parameters_filtering_vi = {
 parameters_filtering_yo = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
+    "cond_remove_words_with_incorrect_substrings": True,
     "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
     "cond_remove_long_words": True,
     "length_word_max_cutoff": 30,
@@ -824,7 +824,7 @@ parameters_filtering_yo = {
     "words_augmentation_join_char": "",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,
@@ -856,9 +856,9 @@ parameters_filtering_zh = {
     "cond_words_augmentation": True,
     "words_augmentation_group_sizes": [2],
     "words_augmentation_join_char": "",
-    "cond_check_stopwords": False,
+    "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
+    "cond_check_flagged_words": True,
     "flagged_words_max_cutoff": 0.2,
     "cond_check_lang_id": True,
     "lang_id_min_cutoff": 0.75,