Merge pull request #26 from XiaoMi/hotfix-postprocessing

修复针对英文切分时空格丢失问题
XiaoMi · Jan 18, 2021 · f8e4aec · f8e4aec
2 parents 7d0c578 + 4fcc5a9
commit f8e4aec
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/minlp-tokenizer/minlptokenizer/tokenizer.py b/minlp-tokenizer/minlptokenizer/tokenizer.py
@@ -125,7 +125,7 @@ def tag2words(self, text, y_pred_result):
                 word = ''
         if word:
             words.append(word)
-        return regex.split(r'\s+', ' '.join(words))
+        return words
 
     def set_interfere_factor(self, interfere_factor):
         """

diff --git a/minlp-tokenizer/requirements.txt b/minlp-tokenizer/requirements.txt
@@ -1,3 +1,3 @@
-tensorflow>=1.15,<2
+tensorflow>=1.14
 pyahocorasick
 regex
diff --git a/minlp-tokenizer/setup.py b/minlp-tokenizer/setup.py
@@ -32,7 +32,7 @@
 
 setup(
     name='minlp-tokenizer',
-    version='3.2.3',
+    version='3.2.4',
     description='MiNLP-Tokenizer中文分词工具',
     author='Yuankai Guo, Liang Shi, Yupeng Chen',
     author_email='[email protected], [email protected]',