Skip to content

Commit

Permalink
fix: [languages] fix items language min probability
Browse files Browse the repository at this point in the history
  • Loading branch information
Terrtia committed Mar 27, 2024
1 parent ee563a7 commit a3a664b
Showing 1 changed file with 7 additions and 3 deletions.
10 changes: 7 additions & 3 deletions bin/lib/Language.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ def delete_obj_translation(obj_global_id, language, field=''):

class LanguagesDetector:

def __init__(self, nb_langs=3, min_proportion=0.2, min_probability=0.7, min_len=0):
def __init__(self, nb_langs=3, min_proportion=0.2, min_probability=-1, min_len=0):
self.lt = LibreTranslateAPI(get_translator_instance())
try:
self.lt.languages()
Expand All @@ -461,9 +461,11 @@ def detect_gcld3(self, content):
languages.append(lang.language)
return languages

def detect_lexilang(self, content): # TODO clean text ??? - TODO REMOVE SEPARATOR
def detect_lexilang(self, content):
language, prob = lexilang_detect(content)
if prob > 0:
if prob > 0 and self.min_probability == -1:
return [language]
elif prob > 0.4:
return [language]
else:
return []
Expand Down Expand Up @@ -506,6 +508,8 @@ def detect(self, content, force_gcld3=False): # TODO detect length between 20-2
# libretranslate
# else:
# languages = self.detect_libretranslate(content)
if not languages:
languages = []
return languages

class LanguageTranslator:
Expand Down

0 comments on commit a3a664b

Please sign in to comment.