diff --git a/CHANGES.rst b/CHANGES.rst index 8d025e03d..a52176b74 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,9 @@ - Fix bug that caused result of ``set_array_compression`` to be overwritten by ``all_array_compression`` argument to ``write_to``. [#510] +- Add workaround for Python OSX write limit bug + (see https://bugs.python.org/issue24658). [#521] + 2.0.1 (2018-05-08) ------------------ diff --git a/asdf/generic_io.py b/asdf/generic_io.py index cd70e6b58..d7d06bc91 100644 --- a/asdf/generic_io.py +++ b/asdf/generic_io.py @@ -107,7 +107,11 @@ def _array_tofile_simple(fd, write, array): if sys.platform == 'darwin': # pragma: no cover def _array_tofile(fd, write, array): - OSX_WRITE_LIMIT = 2 ** 32 + # This value is currently set as a workaround for a known bug in Python + # on OSX. Individual writes must be less than 2GB, which necessitates + # the chunk size here if we want it to remain a power of 2. + # See https://bugs.python.org/issue24658. + OSX_WRITE_LIMIT = 2 ** 30 if fd is None or array.nbytes >= OSX_WRITE_LIMIT and array.nbytes % 4096 == 0: return _array_tofile_chunked(write, array, OSX_WRITE_LIMIT) return _array_tofile_simple(fd, write, array)