Skip to content

Commit

Permalink
Implement basic threshold for inlining array data
Browse files Browse the repository at this point in the history
  • Loading branch information
drdavella committed Oct 9, 2018
1 parent cb23d39 commit 5ceed65
Showing 1 changed file with 10 additions and 2 deletions.
12 changes: 10 additions & 2 deletions asdf/block.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
from . import yamlutil


_DEFAULT_INLINE_THRESHOLD_SIZE = 100


class BlockManager(object):
"""
Manages the `Block`s associated with a ASDF file.
Expand All @@ -44,6 +47,8 @@ def __init__(self, asdffile, copy_arrays=False):
'streamed': self._streamed_blocks
}

self._inline_threshold_size = _DEFAULT_INLINE_THRESHOLD_SIZE

self._data_to_block_mapping = {}
self._validate_checksums = False
self._memmap = not copy_arrays
Expand Down Expand Up @@ -702,8 +707,7 @@ def find_or_create_block_for_array(self, arr, ctx):
block : Block
"""
from .tags.core import ndarray
if (isinstance(arr, ndarray.NDArrayType) and
arr.block is not None):
if (isinstance(arr, ndarray.NDArrayType) and arr.block is not None):
if arr.block in self.blocks:
return arr.block
else:
Expand All @@ -714,6 +718,10 @@ def find_or_create_block_for_array(self, arr, ctx):
if block is not None:
return block
block = Block(base)

if arr.size <= self._inline_threshold_size:
block._array_storage = 'inline'

self.add(block)
self._handle_global_block_settings(ctx, block)
return block
Expand Down

0 comments on commit 5ceed65

Please sign in to comment.