Add absolute confidence metric based on unique and most common ngrams

Cargo.toml

@@ -63,81 +63,81 @@ serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.134"
 strum = "0.26.3"
 strum_macros = "0.26.4"
-lingua-afrikaans-language-model = { path = "language-models/af", version = "1.1.0", optional = true }
-lingua-albanian-language-model = { path = "language-models/sq", version = "1.1.0", optional = true }
-lingua-arabic-language-model = { path = "language-models/ar", version = "1.1.0", optional = true }
-lingua-armenian-language-model = { path = "language-models/hy", version = "1.1.0", optional = true }
-lingua-azerbaijani-language-model = { path = "language-models/az", version = "1.1.0", optional = true }
-lingua-basque-language-model = { path = "language-models/eu", version = "1.1.0", optional = true }
-lingua-belarusian-language-model = { path = "language-models/be", version = "1.1.0", optional = true }
-lingua-bengali-language-model = { path = "language-models/bn", version = "1.1.0", optional = true }
-lingua-bokmal-language-model = { path = "language-models/nb", version = "1.1.0", optional = true }
-lingua-bosnian-language-model = { path = "language-models/bs", version = "1.1.0", optional = true }
-lingua-bulgarian-language-model = { path = "language-models/bg", version = "1.1.0", optional = true }
-lingua-catalan-language-model = { path = "language-models/ca", version = "1.1.0", optional = true }
-lingua-chinese-language-model = { path = "language-models/zh", version = "1.1.0", optional = true }
-lingua-croatian-language-model = { path = "language-models/hr", version = "1.1.0", optional = true }
-lingua-czech-language-model = { path = "language-models/cs", version = "1.1.0", optional = true }
-lingua-danish-language-model = { path = "language-models/da", version = "1.1.0", optional = true }
-lingua-dutch-language-model = { path = "language-models/nl", version = "1.1.0", optional = true }
-lingua-english-language-model = { path = "language-models/en", version = "1.1.0", optional = true }
-lingua-esperanto-language-model = { path = "language-models/eo", version = "1.1.0", optional = true }
-lingua-estonian-language-model = { path = "language-models/et", version = "1.1.0", optional = true }
-lingua-finnish-language-model = { path = "language-models/fi", version = "1.1.0", optional = true }
-lingua-french-language-model = { path = "language-models/fr", version = "1.1.0", optional = true }
-lingua-ganda-language-model = { path = "language-models/lg", version = "1.1.0", optional = true }
-lingua-georgian-language-model = { path = "language-models/ka", version = "1.1.0", optional = true }
-lingua-german-language-model = { path = "language-models/de", version = "1.1.0", optional = true }
-lingua-greek-language-model = { path = "language-models/el", version = "1.1.0", optional = true }
-lingua-gujarati-language-model = { path = "language-models/gu", version = "1.1.0", optional = true }
-lingua-hebrew-language-model = { path = "language-models/he", version = "1.1.0", optional = true }
-lingua-hindi-language-model = { path = "language-models/hi", version = "1.1.0", optional = true }
-lingua-hungarian-language-model = { path = "language-models/hu", version = "1.1.0", optional = true }
-lingua-icelandic-language-model = { path = "language-models/is", version = "1.1.0", optional = true }
-lingua-indonesian-language-model = { path = "language-models/id", version = "1.1.0", optional = true }
-lingua-irish-language-model = { path = "language-models/ga", version = "1.1.0", optional = true }
-lingua-italian-language-model = { path = "language-models/it", version = "1.1.0", optional = true }
-lingua-japanese-language-model = { path = "language-models/ja", version = "1.1.0", optional = true }
-lingua-kazakh-language-model = { path = "language-models/kk", version = "1.1.0", optional = true }
-lingua-korean-language-model = { path = "language-models/ko", version = "1.1.0", optional = true }
-lingua-latin-language-model = { path = "language-models/la", version = "1.1.0", optional = true }
-lingua-latvian-language-model = { path = "language-models/lv", version = "1.1.0", optional = true }
-lingua-lithuanian-language-model = { path = "language-models/lt", version = "1.1.0", optional = true }
-lingua-macedonian-language-model = { path = "language-models/mk", version = "1.1.0", optional = true }
-lingua-malay-language-model = { path = "language-models/ms", version = "1.1.0", optional = true }
-lingua-maori-language-model = { path = "language-models/mi", version = "1.1.0", optional = true }
-lingua-marathi-language-model = { path = "language-models/mr", version = "1.1.0", optional = true }
-lingua-mongolian-language-model = { path = "language-models/mn", version = "1.1.0", optional = true }
-lingua-nynorsk-language-model = { path = "language-models/nn", version = "1.1.0", optional = true }
-lingua-persian-language-model = { path = "language-models/fa", version = "1.1.0", optional = true }
-lingua-polish-language-model = { path = "language-models/pl", version = "1.1.0", optional = true }
-lingua-portuguese-language-model = { path = "language-models/pt", version = "1.1.0", optional = true }
-lingua-punjabi-language-model = { path = "language-models/pa", version = "1.1.0", optional = true }
-lingua-romanian-language-model = { path = "language-models/ro", version = "1.1.0", optional = true }
-lingua-russian-language-model = { path = "language-models/ru", version = "1.1.0", optional = true }
-lingua-serbian-language-model = { path = "language-models/sr", version = "1.1.0", optional = true }
-lingua-shona-language-model = { path = "language-models/sn", version = "1.1.0", optional = true }
-lingua-slovak-language-model = { path = "language-models/sk", version = "1.1.0", optional = true }
-lingua-slovene-language-model = { path = "language-models/sl", version = "1.1.0", optional = true }
-lingua-somali-language-model = { path = "language-models/so", version = "1.1.0", optional = true }
-lingua-sotho-language-model = { path = "language-models/st", version = "1.1.0", optional = true }
-lingua-spanish-language-model = { path = "language-models/es", version = "1.1.0", optional = true }
-lingua-swahili-language-model = { path = "language-models/sw", version = "1.1.0", optional = true }
-lingua-swedish-language-model = { path = "language-models/sv", version = "1.1.0", optional = true }
-lingua-tagalog-language-model = { path = "language-models/tl", version = "1.1.0", optional = true }
-lingua-tamil-language-model = { path = "language-models/ta", version = "1.1.0", optional = true }
-lingua-telugu-language-model = { path = "language-models/te", version = "1.1.0", optional = true }
-lingua-thai-language-model = { path = "language-models/th", version = "1.1.0", optional = true }
-lingua-tsonga-language-model = { path = "language-models/ts", version = "1.1.0", optional = true }
-lingua-tswana-language-model = { path = "language-models/tn", version = "1.1.0", optional = true }
-lingua-turkish-language-model = { path = "language-models/tr", version = "1.1.0", optional = true }
-lingua-ukrainian-language-model = { path = "language-models/uk", version = "1.1.0", optional = true }
-lingua-urdu-language-model = { path = "language-models/ur", version = "1.1.0", optional = true }
-lingua-vietnamese-language-model = { path = "language-models/vi", version = "1.1.0", optional = true }
-lingua-welsh-language-model = { path = "language-models/cy", version = "1.1.0", optional = true }
-lingua-xhosa-language-model = { path = "language-models/xh", version = "1.1.0", optional = true }
-lingua-yoruba-language-model = { path = "language-models/yo", version = "1.1.0", optional = true }
-lingua-zulu-language-model = { path = "language-models/zu", version = "1.1.0", optional = true }
+lingua-afrikaans-language-model = { path = "language-models/af", version = "1.2.0", optional = true }
+lingua-albanian-language-model = { path = "language-models/sq", version = "1.2.0", optional = true }
+lingua-arabic-language-model = { path = "language-models/ar", version = "1.2.0", optional = true }
+lingua-armenian-language-model = { path = "language-models/hy", version = "1.2.0", optional = true }
+lingua-azerbaijani-language-model = { path = "language-models/az", version = "1.2.0", optional = true }
+lingua-basque-language-model = { path = "language-models/eu", version = "1.2.0", optional = true }
+lingua-belarusian-language-model = { path = "language-models/be", version = "1.2.0", optional = true }
+lingua-bengali-language-model = { path = "language-models/bn", version = "1.2.0", optional = true }
+lingua-bokmal-language-model = { path = "language-models/nb", version = "1.2.0", optional = true }
+lingua-bosnian-language-model = { path = "language-models/bs", version = "1.2.0", optional = true }
+lingua-bulgarian-language-model = { path = "language-models/bg", version = "1.2.0", optional = true }
+lingua-catalan-language-model = { path = "language-models/ca", version = "1.2.0", optional = true }
+lingua-chinese-language-model = { path = "language-models/zh", version = "1.2.0", optional = true }
+lingua-croatian-language-model = { path = "language-models/hr", version = "1.2.0", optional = true }
+lingua-czech-language-model = { path = "language-models/cs", version = "1.2.0", optional = true }
+lingua-danish-language-model = { path = "language-models/da", version = "1.2.0", optional = true }
+lingua-dutch-language-model = { path = "language-models/nl", version = "1.2.0", optional = true }
+lingua-english-language-model = { path = "language-models/en", version = "1.2.0", optional = true }
+lingua-esperanto-language-model = { path = "language-models/eo", version = "1.2.0", optional = true }
+lingua-estonian-language-model = { path = "language-models/et", version = "1.2.0", optional = true }
+lingua-finnish-language-model = { path = "language-models/fi", version = "1.2.0", optional = true }
+lingua-french-language-model = { path = "language-models/fr", version = "1.2.0", optional = true }
+lingua-ganda-language-model = { path = "language-models/lg", version = "1.2.0", optional = true }
+lingua-georgian-language-model = { path = "language-models/ka", version = "1.2.0", optional = true }
+lingua-german-language-model = { path = "language-models/de", version = "1.2.0", optional = true }
+lingua-greek-language-model = { path = "language-models/el", version = "1.2.0", optional = true }
+lingua-gujarati-language-model = { path = "language-models/gu", version = "1.2.0", optional = true }
+lingua-hebrew-language-model = { path = "language-models/he", version = "1.2.0", optional = true }
+lingua-hindi-language-model = { path = "language-models/hi", version = "1.2.0", optional = true }
+lingua-hungarian-language-model = { path = "language-models/hu", version = "1.2.0", optional = true }
+lingua-icelandic-language-model = { path = "language-models/is", version = "1.2.0", optional = true }
+lingua-indonesian-language-model = { path = "language-models/id", version = "1.2.0", optional = true }
+lingua-irish-language-model = { path = "language-models/ga", version = "1.2.0", optional = true }
+lingua-italian-language-model = { path = "language-models/it", version = "1.2.0", optional = true }
+lingua-japanese-language-model = { path = "language-models/ja", version = "1.2.0", optional = true }
+lingua-kazakh-language-model = { path = "language-models/kk", version = "1.2.0", optional = true }
+lingua-korean-language-model = { path = "language-models/ko", version = "1.2.0", optional = true }
+lingua-latin-language-model = { path = "language-models/la", version = "1.2.0", optional = true }
+lingua-latvian-language-model = { path = "language-models/lv", version = "1.2.0", optional = true }
+lingua-lithuanian-language-model = { path = "language-models/lt", version = "1.2.0", optional = true }
+lingua-macedonian-language-model = { path = "language-models/mk", version = "1.2.0", optional = true }
+lingua-malay-language-model = { path = "language-models/ms", version = "1.2.0", optional = true }
+lingua-maori-language-model = { path = "language-models/mi", version = "1.2.0", optional = true }
+lingua-marathi-language-model = { path = "language-models/mr", version = "1.2.0", optional = true }
+lingua-mongolian-language-model = { path = "language-models/mn", version = "1.2.0", optional = true }
+lingua-nynorsk-language-model = { path = "language-models/nn", version = "1.2.0", optional = true }
+lingua-persian-language-model = { path = "language-models/fa", version = "1.2.0", optional = true }
+lingua-polish-language-model = { path = "language-models/pl", version = "1.2.0", optional = true }
+lingua-portuguese-language-model = { path = "language-models/pt", version = "1.2.0", optional = true }
+lingua-punjabi-language-model = { path = "language-models/pa", version = "1.2.0", optional = true }
+lingua-romanian-language-model = { path = "language-models/ro", version = "1.2.0", optional = true }
+lingua-russian-language-model = { path = "language-models/ru", version = "1.2.0", optional = true }
+lingua-serbian-language-model = { path = "language-models/sr", version = "1.2.0", optional = true }
+lingua-shona-language-model = { path = "language-models/sn", version = "1.2.0", optional = true }
+lingua-slovak-language-model = { path = "language-models/sk", version = "1.2.0", optional = true }
+lingua-slovene-language-model = { path = "language-models/sl", version = "1.2.0", optional = true }
+lingua-somali-language-model = { path = "language-models/so", version = "1.2.0", optional = true }
+lingua-sotho-language-model = { path = "language-models/st", version = "1.2.0", optional = true }
+lingua-spanish-language-model = { path = "language-models/es", version = "1.2.0", optional = true }
+lingua-swahili-language-model = { path = "language-models/sw", version = "1.2.0", optional = true }
+lingua-swedish-language-model = { path = "language-models/sv", version = "1.2.0", optional = true }
+lingua-tagalog-language-model = { path = "language-models/tl", version = "1.2.0", optional = true }
+lingua-tamil-language-model = { path = "language-models/ta", version = "1.2.0", optional = true }
+lingua-telugu-language-model = { path = "language-models/te", version = "1.2.0", optional = true }
+lingua-thai-language-model = { path = "language-models/th", version = "1.2.0", optional = true }
+lingua-tsonga-language-model = { path = "language-models/ts", version = "1.2.0", optional = true }
+lingua-tswana-language-model = { path = "language-models/tn", version = "1.2.0", optional = true }
+lingua-turkish-language-model = { path = "language-models/tr", version = "1.2.0", optional = true }
+lingua-ukrainian-language-model = { path = "language-models/uk", version = "1.2.0", optional = true }
+lingua-urdu-language-model = { path = "language-models/ur", version = "1.2.0", optional = true }
+lingua-vietnamese-language-model = { path = "language-models/vi", version = "1.2.0", optional = true }
+lingua-welsh-language-model = { path = "language-models/cy", version = "1.2.0", optional = true }
+lingua-xhosa-language-model = { path = "language-models/xh", version = "1.2.0", optional = true }
+lingua-yoruba-language-model = { path = "language-models/yo", version = "1.2.0", optional = true }
+lingua-zulu-language-model = { path = "language-models/zu", version = "1.2.0", optional = true }
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 ahash = "0.8.11"

language-models/af/Cargo.toml

@@ -14,7 +14,7 @@
 
 [package]
 name = "lingua-afrikaans-language-model"
-version = "1.1.0"
+version = "1.2.0"
 authors = ["Peter M. Stahl <pemistahl@gmail.com>"]
 description = """
 The Afrikaans language model for Lingua, an accurate natural language detection library
@@ -34,4 +34,4 @@ keywords = [
 ]
 
 [dependencies]
-include_dir = "0.7.3"
+include_dir = "0.7.4"

language-models/af/README.md

@@ -6,6 +6,12 @@ the most accurate natural language detection library in the Rust ecosystem.
 
 ### Changelog
 
+#### Version 1.2.0
+
+- The language model has been enhanced by including unique and most common
+  ngrams to support an absolute confidence metric which is independent of
+  other languages.
+
 #### Version 1.1.0
 
 - The language model files are now compressed with the Brotli algorithm which

language-models/af/models/unique_trigrams.json.br

@@ -0,0 +1,6 @@
+@,ÌóÝ<M
+®–ƒ0g†x1sÆo$?¥Q{ÇáöƒÒMá!3÷[è«eKKd+AXúJ§‹àkÇ÷O;Šu¡ÕK0ÆH¸þüÂmìäxQ¨DÕö/yoø:ïñÝ>ÍùÊâÅH[æ•AˆO8;×~m(í×)ÙÞÛL|ôM2‹
+f¹‚Jéyî…ãaáwÊسˆ”@çbàp{"Ö$nKܗˆd¢àš€(d3_ø¯¢ÃAŸÃ<±J¦FvTðëGª*QJZ«®ÑØ"@²ƒ»þú엑ªï©Õċ'>8lr
+Y$Î¥o
+e8$9ƒ€R©ÂÄç­
+û;É»š0c¤ºSyC¿Îabh#­¿iƳ(ԙ+r‡›d™o––ÓyË!*!!Cꍒt”áß÷Ãí

language-models/af/models/unique_unigrams.json.br

@@ -0,0 +1 @@
+€{"language": "AFRIKAANS", "ngrams": ["ȇ", "ȅ", "ʼn"]}

language-models/ar/Cargo.toml

@@ -14,7 +14,7 @@
 
 [package]
 name = "lingua-arabic-language-model"
-version = "1.1.0"
+version = "1.2.0"
 authors = ["Peter M. Stahl <pemistahl@gmail.com>"]
 description = """
 The Arabic language model for Lingua, an accurate natural language detection library
@@ -34,4 +34,4 @@ keywords = [
 ]
 
 [dependencies]
-include_dir = "0.7.3"
+include_dir = "0.7.4"

language-models/ar/README.md

@@ -6,6 +6,12 @@ the most accurate natural language detection library in the Rust ecosystem.
 
 ### Changelog
 
+#### Version 1.2.0
+
+- The language model has been enhanced by including unique and most common
+  ngrams to support an absolute confidence metric which is independent of
+  other languages.
+
 #### Version 1.1.0
 
 - The language model files are now compressed with the Brotli algorithm which

language-models/ar/models/mostcommon_quadrigrams.json.br

@@ -0,0 +1,3 @@
+M
@<˜s¯V!Ú:Xi	O=—O–¿ÆÎÙ«n¹Ë&ÇRÇÃÑsËRd«­³Yº‚¦ŽT_€”ÐPŠ%e˜Üܺ³¶‚²Cmm.P
+í[Ú¾S
+,Ãô†ÍÓDyÐK>X¹T[^s½g©"î;wtE½^!„‘AQ¡ ~ Ž(XÉúÕ]5dÀ/š€&r݁ÖÙ"qmî

language-models/ar/models/mostcommon_trigrams.json.br

@@ -0,0 +1,2 @@
+
 <h—×KØÄؘx `ݺOŒþ:¯nó¸ÿX¡=oÈDŠ{L‘­¶f	êʖ:R}
RBC)””arsê¶z>øÅ.Öbmk
+¦pºÍš-3
n^úÀr˜[þ•ú(VÄ%°ä%µ(®ßµÛÏ`óR33ŒTâ†X¡½,FBz‡_È{

language-models/az/Cargo.toml

@@ -14,7 +14,7 @@
 
 [package]
 name = "lingua-azerbaijani-language-model"
-version = "1.1.0"
+version = "1.2.0"
 authors = ["Peter M. Stahl <pemistahl@gmail.com>"]
 description = """
 The Azerbaijani language model for Lingua, an accurate natural language detection library
@@ -34,4 +34,4 @@ keywords = [
 ]
 
 [dependencies]
-include_dir = "0.7.3"
+include_dir = "0.7.4"

language-models/az/README.md

@@ -6,6 +6,12 @@ the most accurate natural language detection library in the Rust ecosystem.
 
 ### Changelog
 
+#### Version 1.2.0
+
+- The language model has been enhanced by including unique and most common
+  ngrams to support an absolute confidence metric which is independent of
+  other languages.
+
 #### Version 1.1.0
 
 - The language model files are now compressed with the Brotli algorithm which

language-models/az/models/mostcommon_quadrigrams.json.br

@@ -0,0 +1 @@
+
@œv,ž´	ÖçâTݔ¯r#A@‚Z2Ē½bÇ<ïŽxf~Æ&‹NW€Xî@Ú[óôAìﷇo­µ$i¥Û"æõ–Ù¤ñ¢ùAs=)›ì0J©µ½bJg§ÓÊ	Íqw‚Ç4Å(3KÈCSZ›vësj±wq[)G‹hš-®?`ÿ3ßé´ðÊÁîmŸ_6åVü

language-models/az/models/unique_bigrams.json.br

@@ -0,0 +1,4 @@
+ ,x“/dÁb5*üQqšbsj=%
+Ä^æ7sm¥-—¶2Ä+ã…ñ—"[mN9ò8 ¬¶C‡ÃD«Ï¬åüæ;=÷T¡–Ù„ÒŠp
+uS"úèÚØ¥²Õ…ýCüš»¾‚äúcðôˀƒ;cƒP°ß¨›‹+…l!Ý©ü0& x˜-„>qì„*˜%þ†XBðîmbUsNºcdCSaߙÔJJOcåfÿçÒèf4°‡‘¾…a!è«Ì݇GŽ®
+¹[etǾ÷niQ<ÇL˜Ø*¶%4‰‡‘Ì#s
K£uxn$lò./ù<{‰vëŸÊL-üA\3¬X4=IV‡r7ãäئÇBøú<ÆB0&+äÍÃO,UƇĜ{ùû±lN:Xˆª»ÁCÁÌ

language-models/be/Cargo.toml

@@ -14,7 +14,7 @@
 
 [package]
 name = "lingua-belarusian-language-model"
-version = "1.1.0"
+version = "1.2.0"
 authors = ["Peter M. Stahl <pemistahl@gmail.com>"]
 description = """
 The Belarusian language model for Lingua, an accurate natural language detection library
@@ -34,4 +34,4 @@ keywords = [
 ]
 
 [dependencies]
-include_dir = "0.7.3"
+include_dir = "0.7.4"

language-models/be/README.md

@@ -6,6 +6,12 @@ the most accurate natural language detection library in the Rust ecosystem.
 
 ### Changelog
 
+#### Version 1.2.0
+
+- The language model has been enhanced by including unique and most common
+  ngrams to support an absolute confidence metric which is independent of
+  other languages.
+
 #### Version 1.1.0
 
 - The language model files are now compressed with the Brotli algorithm which

language-models/be/models/mostcommon_trigrams.json.br

@@ -0,0 +1 @@
+
 œ¶l;ÄáX0ÿÀbi¸‹<(«-hkA-ýÇßøº-äzGÐü’‡^DÏ|ŠlµDÞÖŸ˜"™™Íí'” h‚ T¡
xkÇK$ç£>$¥ö<÷jy„(JÏý}†T@ˆ,VS
SÎa7ÚUժĘôöÀË•¸….RÕA²•œ39®5+_	5žØ>?

language-models/bg/Cargo.toml

@@ -14,7 +14,7 @@
 
 [package]
 name = "lingua-bulgarian-language-model"
-version = "1.1.0"
+version = "1.2.0"
 authors = ["Peter M. Stahl <pemistahl@gmail.com>"]
 description = """
 The Bulgarian language model for Lingua, an accurate natural language detection library
@@ -34,4 +34,4 @@ keywords = [
 ]
 
 [dependencies]
-include_dir = "0.7.3"
+include_dir = "0.7.4"