Moved urls to module constants for pretrained embedding utils.

microsoft · May 7, 2019 · 65b76ff · 65b76ff
1 parent 90a1209
commit 65b76ff
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 9 deletions.
diff --git a/utils_nlp/pretrained_embeddings/__init__.py b/utils_nlp/pretrained_embeddings/__init__.py
@@ -1,2 +1,6 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+
+WORD2VEC_URL = 'https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz'
+FASTTEXT_EN_URL = 'https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip'
+GLOVE_URL = 'http://nlp.stanford.edu/data/glove.840B.300d.zip'
diff --git a/utils_nlp/pretrained_embeddings/fasttext.py b/utils_nlp/pretrained_embeddings/fasttext.py
@@ -7,6 +7,7 @@
 from gensim.models.fasttext import load_facebook_model
 
 from utils_nlp.dataset.url_utils import maybe_download
+from utils_nlp.pretrained_embeddings import FASTTEXT_EN_URL
 
 
 def _extract_fasttext_vectors(zip_path, dest_path="."):
@@ -44,8 +45,9 @@ def _download_fasttext_vectors(download_dir, file_name="wiki.en.zip"):
         str: file_path to the downloaded vectors.
     """
 
-    url = "https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip"
-    return maybe_download(url, filename=file_name, work_directory=download_dir)
+    return maybe_download(
+        FASTTEXT_EN_URL, filename=file_name, work_directory=download_dir
+    )
 
 
 def _maybe_download_and_extract(dest_path, file_name):

diff --git a/utils_nlp/pretrained_embeddings/glove.py b/utils_nlp/pretrained_embeddings/glove.py
@@ -9,6 +9,7 @@
 from gensim.test.utils import get_tmpfile
 
 from utils_nlp.dataset.url_utils import maybe_download
+from utils_nlp.pretrained_embeddings import GLOVE_URL
 
 
 def _extract_glove_vectors(zip_path, dest_path="."):
@@ -46,8 +47,7 @@ def _download_glove_vectors(download_dir, file_name="glove.840B.300d.zip"):
         str: file_path to the downloaded vectors.
     """
 
-    url = "http://nlp.stanford.edu/data/glove.840B.300d.zip"
-    return maybe_download(url, filename=file_name, work_directory=download_dir)
+    return maybe_download(GLOVE_URL, filename=file_name, work_directory=download_dir)
 
 
 def _maybe_download_and_extract(dest_path, file_name):

diff --git a/utils_nlp/pretrained_embeddings/word2vec.py b/utils_nlp/pretrained_embeddings/word2vec.py
@@ -7,6 +7,7 @@
 from gensim.models.keyedvectors import KeyedVectors
 
 from utils_nlp.dataset.url_utils import maybe_download
+from utils_nlp.pretrained_embeddings import WORD2VEC_URL
 
 
 def _extract_word2vec_vectors(zip_path, dest_filepath):
@@ -44,11 +45,7 @@ def _download_word2vec_vectors(
         str: file_path to the downloaded vectors.
     """
 
-    url = (
-        "https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300"
-        ".bin.gz "
-    )
-    return maybe_download(url, filename=file_name, work_directory=download_dir)
+    return maybe_download(WORD2VEC_URL, filename=file_name, work_directory=download_dir)
 
 
 def _maybe_download_and_extract(dest_path, file_name):