Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Respect encoding when reading binary keyed vectors #3309

Merged
merged 4 commits into from
Apr 15, 2022
Merged

Respect encoding when reading binary keyed vectors #3309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c59a412
Respect encoding when reading binary keyed vectors
alhoo Mar 20, 2022
998074e
Update keyedvectors.py
piskvorky Apr 15, 2022
662e380
Update test_translation_matrix.py
piskvorky Apr 15, 2022
17c6cf0
Merge branch 'develop' into feature/respect-encoding-when-reading-bin…
piskvorky Apr 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions gensim/models/keyedvectors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1927,7 +1927,7 @@ def _add_word_to_kv(kv, counts, word, weights, vocab_size):
kv.set_vecattr(word, 'count', word_count)


def _add_bytes_to_kv(kv, counts, chunk, vocab_size, vector_size, datatype, unicode_errors):
def _add_bytes_to_kv(kv, counts, chunk, vocab_size, vector_size, datatype, unicode_errors, encoding):
start = 0
processed_words = 0
bytes_per_vector = vector_size * dtype(REAL).itemsize
Expand All @@ -1940,7 +1940,7 @@ def _add_bytes_to_kv(kv, counts, chunk, vocab_size, vector_size, datatype, unico
if i_space == -1 or (len(chunk) - i_vector) < bytes_per_vector:
break

word = chunk[start:i_space].decode("utf-8", errors=unicode_errors)
word = chunk[start:i_space].decode(encoding, errors=unicode_errors)
# Some binary files are reported to have obsolete new line in the beginning of word, remove it
word = word.lstrip('\n')
vector = frombuffer(chunk, offset=i_vector, count=vector_size, dtype=REAL).astype(datatype)
Expand All @@ -1951,15 +1951,18 @@ def _add_bytes_to_kv(kv, counts, chunk, vocab_size, vector_size, datatype, unico
return processed_words, chunk[start:]


def _word2vec_read_binary(fin, kv, counts, vocab_size, vector_size, datatype, unicode_errors, binary_chunk_size):
def _word2vec_read_binary(
fin, kv, counts, vocab_size, vector_size, datatype, unicode_errors, binary_chunk_size,
encoding="utf-8",
):
chunk = b''
tot_processed_words = 0

while tot_processed_words < vocab_size:
new_chunk = fin.read(binary_chunk_size)
chunk += new_chunk
processed_words, chunk = _add_bytes_to_kv(
kv, counts, chunk, vocab_size, vector_size, datatype, unicode_errors)
kv, counts, chunk, vocab_size, vector_size, datatype, unicode_errors, encoding)
tot_processed_words += processed_words
if len(new_chunk) < binary_chunk_size:
break
Expand Down Expand Up @@ -2064,7 +2067,7 @@ def _load_word2vec_format(

if binary:
_word2vec_read_binary(
fin, kv, counts, vocab_size, vector_size, datatype, unicode_errors, binary_chunk_size,
fin, kv, counts, vocab_size, vector_size, datatype, unicode_errors, binary_chunk_size, encoding
)
else:
_word2vec_read_text(fin, kv, counts, vocab_size, vector_size, datatype, unicode_errors, encoding)
Expand Down
1 change: 1 addition & 0 deletions gensim/test/test_translation_matrix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env python
# encoding: utf-8

from collections import namedtuple
import unittest
import logging
Expand Down