Feature/synced collection/reorg (#481)

Reorganizes the package structure so that the synced_collections subpackage is now at the package root and is internally structured with subpackages for data types, backends, and buffers.

* Move synced collections from core to package root.

* Fix all import locations.

* Reorganize internals of synced collection package.

* Fix all imports for reorganized package.

* Hide caching module since it's still experimental.

* Address PR comments.

* Make __all__ into an empty list not rather than a list containing an empty string.

signac/__init__.py

@@ -29,11 +29,11 @@
 from .contrib.job import get_buffer_load, get_buffer_size, set_buffer_size
 from .core.h5store import H5Store, H5StoreManager
 from .core.jsondict import flush_all as flush
-from .core.synced_collections.buffered_collection import buffer_all as buffered
-from .core.synced_collections.buffered_collection import is_buffered
-from .core.synced_collections.collection_json import JSONDict
 from .db import get_database
 from .diff import diff_jobs
+from .synced_collections.backends.collection_json import JSONDict
+from .synced_collections.buffers.buffered_collection import buffer_all as buffered
+from .synced_collections.buffers.buffered_collection import is_buffered
 from .version import __version__
 
 __all__ = [

signac/contrib/hashing.py

@@ -6,7 +6,7 @@
 import hashlib
 import json
 
-from ..core.synced_collections.utils import SyncedCollectionJSONEncoder
+from ..synced_collections.utils import SyncedCollectionJSONEncoder
 
 # We must use the standard library json for exact consistency in formatting
 

signac/contrib/job.py

@@ -13,9 +13,9 @@
 from deprecation import deprecated
 
 from ..core.h5store import H5StoreManager
-from ..core.synced_collections.collection_json import BufferedJSONDict, JSONDict
 from ..errors import KeyTypeError
 from ..sync import sync_jobs
+from ..synced_collections.backends.collection_json import BufferedJSONDict, JSONDict
 from ..version import __version__
 from .errors import DestinationExistsError, JobsCorruptedError
 from .hashing import calc_id

signac/contrib/project.py

@@ -26,8 +26,8 @@
 
 from ..common.config import Config, get_config, load_config
 from ..core.h5store import H5StoreManager
-from ..core.synced_collections.collection_json import BufferedJSONDict
 from ..sync import sync_projects
+from ..synced_collections.backends.collection_json import BufferedJSONDict
 from ..version import SCHEMA_VERSION, __version__
 from .collection import Collection
 from .errors import (

signac/errors.py

@@ -17,6 +17,7 @@
 )
 from .core.errors import Error
 from .core.jsondict import BufferedFileError, BufferException
+from .synced_collections.errors import InvalidKeyError, KeyTypeError
 
 
 class SyncConflict(Error, RuntimeError):
@@ -58,14 +59,6 @@ def __str__(self):
         return "The synchronization failed, because of a schema conflict."
 
 
-class InvalidKeyError(ValueError):
-    """Raised when a user uses a non-conforming key."""
-
-
-class KeyTypeError(TypeError):
-    """Raised when a user uses a key of invalid type."""
-
-
 __all__ = [
     "AuthenticationError",
     "BufferException",
@@ -80,6 +73,7 @@ class KeyTypeError(TypeError):
     "IncompatibleSchemaVersion",
     "InvalidKeyError",
     "JobsCorruptedError",
+    "KeyTypeError",
     "SchemaSyncConflict",
     "StatepointParsingError",
     "SyncConflict",

signac/core/synced_collections/__init__.py → signac/synced_collections/__init__.py

@@ -19,3 +19,7 @@
 `JSONDict` initialized pointing to a particular file can be modified like a normal
 dict, and all changes will be automatically persisted to a JSON file.
 """
+
+from .data_types import SyncedAttrDict, SyncedCollection, SyncedList
+
+__all__ = ["SyncedCollection", "SyncedAttrDict", "SyncedList"]

signac/synced_collections/backends/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) 2020 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""This subpackage defines supported backends.
+
+No backends are imported by default. Users should import desired backends as
+needed.
+"""
+from typing import List as _List
+
+__all__: _List[str] = []

signac/core/synced_collections/collection_json.py → signac/synced_collections/backends/collection_json.py

@@ -10,13 +10,13 @@
 import warnings
 from typing import Tuple
 
-from .memory_buffered_collection import SharedMemoryFileBufferedCollection
-from .serialized_file_buffered_collection import SerializedFileBufferedCollection
-from .synced_attr_dict import SyncedAttrDict
-from .synced_collection import SyncedCollection
-from .synced_list import SyncedList
-from .utils import SyncedCollectionJSONEncoder
-from .validators import json_format_validator
+from .. import SyncedAttrDict, SyncedCollection, SyncedList
+from ..buffers.memory_buffered_collection import SharedMemoryFileBufferedCollection
+from ..buffers.serialized_file_buffered_collection import (
+    SerializedFileBufferedCollection,
+)
+from ..utils import SyncedCollectionJSONEncoder
+from ..validators import json_format_validator
 
 
 # TODO: This method should be removed in signac 2.0.

signac/core/synced_collections/collection_mongodb.py → signac/synced_collections/backends/collection_mongodb.py

@@ -2,9 +2,7 @@
 # All rights reserved.
 # This software is licensed under the BSD 3-Clause License.
 """Implements a MongoDB SyncedCollection backend."""
-from .synced_attr_dict import SyncedAttrDict
-from .synced_collection import SyncedCollection
-from .synced_list import SyncedList
+from .. import SyncedAttrDict, SyncedCollection, SyncedList
 
 try:
     import bson

signac/core/synced_collections/collection_redis.py → signac/synced_collections/backends/collection_redis.py

@@ -4,9 +4,7 @@
 """Implements a Redis SyncedCollection backend."""
 import json
 
-from .synced_attr_dict import SyncedAttrDict
-from .synced_collection import SyncedCollection
-from .synced_list import SyncedList
+from .. import SyncedAttrDict, SyncedCollection, SyncedList
 
 
 class RedisCollection(SyncedCollection):

signac/core/synced_collections/collection_zarr.py → signac/synced_collections/backends/collection_zarr.py

@@ -4,9 +4,7 @@
 """Implements a Zarr SyncedCollection backend."""
 from copy import deepcopy
 
-from .synced_attr_dict import SyncedAttrDict
-from .synced_collection import SyncedCollection
-from .synced_list import SyncedList
+from .. import SyncedAttrDict, SyncedCollection, SyncedList
 
 try:
     import numcodecs

signac/synced_collections/buffers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2020 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""Defines the buffering protocol for synced collections.
+
+In addition to defining the buffering protocol for synced collections in
+:class:`~.BufferedCollection`, this subpackage also defines a number of
+supported buffering implementations. No buffers are imported by default. Users
+should import desired buffers as needed.
+"""
+from typing import List as _List
+
+__all__: _List[str] = []

signac/core/synced_collections/buffered_collection.py → signac/synced_collections/buffers/buffered_collection.py

@@ -40,9 +40,9 @@
 from inspect import isabstract
 from typing import Any, List
 
-from .errors import BufferedError
-from .synced_collection import SyncedCollection
-from .utils import _CounterFuncContext
+from .. import SyncedCollection
+from ..errors import BufferedError
+from ..utils import _CounterFuncContext
 
 logger = logging.getLogger(__name__)
 

signac/core/synced_collections/file_buffered_collection.py → signac/synced_collections/buffers/file_buffered_collection.py

@@ -15,10 +15,10 @@
 from threading import RLock
 from typing import Dict, Tuple, Union
 
+from ..data_types.synced_collection import _LoadAndSave
+from ..errors import MetadataError
+from ..utils import _NullContext
 from .buffered_collection import BufferedCollection
-from .errors import MetadataError
-from .synced_collection import _LoadAndSave
-from .utils import _NullContext
 
 
 class _BufferedLoadAndSave(_LoadAndSave):

signac/core/synced_collections/memory_buffered_collection.py → signac/synced_collections/buffers/memory_buffered_collection.py

@@ -12,7 +12,7 @@
 more complicated synchronization protocols.
 """
 
-from .errors import MetadataError
+from ..errors import MetadataError
 from .file_buffered_collection import FileBufferedCollection
 
 

signac/core/synced_collections/serialized_file_buffered_collection.py → signac/synced_collections/buffers/serialized_file_buffered_collection.py

@@ -11,9 +11,9 @@
 import hashlib
 import json
 
-from .errors import MetadataError
+from ..errors import MetadataError
+from ..utils import SyncedCollectionJSONEncoder
 from .file_buffered_collection import FileBufferedCollection
-from .utils import SyncedCollectionJSONEncoder
 
 
 class SerializedFileBufferedCollection(FileBufferedCollection):

signac/synced_collections/data_types/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2020 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""This subpackage defines various synced data types."""
+
+from .synced_attr_dict import SyncedAttrDict
+from .synced_collection import SyncedCollection
+from .synced_list import SyncedList
+
+__all__ = ["SyncedCollection", "SyncedAttrDict", "SyncedList"]

signac/core/synced_collections/synced_attr_dict.py → signac/synced_collections/data_types/synced_attr_dict.py

@@ -12,9 +12,9 @@
 from collections.abc import Mapping, MutableMapping
 from typing import Tuple
 
+from ..utils import AbstractTypeResolver
+from ..validators import no_dot_in_key
 from .synced_collection import SyncedCollection, _sc_resolver
-from .utils import AbstractTypeResolver
-from .validators import no_dot_in_key
 
 # Identifies mappings, which are the base type for this class.
 _mapping_resolver = AbstractTypeResolver(

signac/core/synced_collections/synced_collection.py → signac/synced_collections/data_types/synced_collection.py

@@ -9,7 +9,7 @@
 from threading import RLock
 from typing import Any, Callable, DefaultDict, List
 
-from .utils import AbstractTypeResolver, _CounterContext, _NullContext
+from ..utils import AbstractTypeResolver, _CounterContext, _NullContext
 
 try:
     import numpy

signac/core/synced_collections/synced_list.py → signac/synced_collections/data_types/synced_list.py

@@ -9,8 +9,8 @@
 
 from collections.abc import MutableSequence, Sequence
 
+from ..utils import AbstractTypeResolver
 from .synced_collection import NUMPY, SyncedCollection, _sc_resolver
-from .utils import AbstractTypeResolver
 
 if NUMPY:
     import numpy

signac/core/synced_collections/errors.py → signac/synced_collections/errors.py

@@ -4,11 +4,8 @@
 """Errors raised by synced collections."""
 
 
-from ..errors import Error
-
-
-class BufferException(Error):
-    """An exception occured in buffered mode."""
+class BufferException(RuntimeError):
+    """Raised when any exception related to buffering occurs."""
 
 
 class BufferedError(BufferException):
@@ -42,3 +39,11 @@ def __init__(self, filename, contents):
 
     def __str__(self):
         return f"{self.filename} appears to have been externally modified."
+
+
+class KeyTypeError(TypeError):
+    """Raised when a user uses a key of invalid type."""
+
+
+class InvalidKeyError(ValueError):
+    """Raised when a user uses a non-conforming key."""

signac/core/synced_collections/validators.py → signac/synced_collections/validators.py

@@ -12,7 +12,7 @@
 
 from collections.abc import Mapping, Sequence
 
-from ...errors import InvalidKeyError, KeyTypeError
+from .errors import InvalidKeyError, KeyTypeError
 from .utils import AbstractTypeResolver
 
 try:

tests/test_buffered_mode.py

@@ -13,8 +13,8 @@
 from test_project import TestProjectBase
 
 import signac
-from signac.core.synced_collections.errors import BufferedError
 from signac.errors import BufferedFileError, Error
+from signac.synced_collections.errors import BufferedError
 
 PYPY = "PyPy" in platform.python_implementation()
 

tests/test_synced_collections/synced_collection_test.py

@@ -7,8 +7,8 @@
 
 import pytest
 
-from signac.core.synced_collections.synced_collection import SyncedCollection
 from signac.errors import InvalidKeyError, KeyTypeError
+from signac.synced_collections import SyncedCollection
 
 try:
     import numpy

tests/test_synced_collections/test_json_buffered_collection.py

@@ -11,16 +11,16 @@
 import pytest
 from test_json_collection import JSONCollectionTest, TestJSONDict, TestJSONList
 
-from signac.core.synced_collections.buffered_collection import buffer_all
-from signac.core.synced_collections.collection_json import (
+from signac.synced_collections.backends.collection_json import (
     BufferedJSONCollection,
     BufferedJSONDict,
     BufferedJSONList,
     MemoryBufferedJSONCollection,
     MemoryBufferedJSONDict,
     MemoryBufferedJSONList,
 )
-from signac.core.synced_collections.errors import BufferedError, MetadataError
+from signac.synced_collections.buffers.buffered_collection import buffer_all
+from signac.synced_collections.errors import BufferedError, MetadataError
 
 
 class BufferedJSONCollectionTest(JSONCollectionTest):

tests/test_synced_collections/test_json_collection.py

@@ -8,7 +8,7 @@
 import pytest
 from synced_collection_test import SyncedDictTest, SyncedListTest
 
-from signac.core.synced_collections.collection_json import (
+from signac.synced_collections.backends.collection_json import (
     JSONCollection,
     JSONDict,
     JSONList,

tests/test_synced_collections/test_mongodb_collection.py

@@ -4,7 +4,7 @@
 import pytest
 from synced_collection_test import SyncedDictTest, SyncedListTest
 
-from signac.core.synced_collections.collection_mongodb import (
+from signac.synced_collections.backends.collection_mongodb import (
     MongoDBCollection,
     MongoDBDict,
     MongoDBList,

tests/test_synced_collections/test_redis_collection.py

@@ -7,7 +7,7 @@
 import pytest
 from synced_collection_test import SyncedDictTest, SyncedListTest
 
-from signac.core.synced_collections.collection_redis import (
+from signac.synced_collections.backends.collection_redis import (
     RedisCollection,
     RedisDict,
     RedisList,

tests/test_synced_collections/test_utils.py

@@ -8,9 +8,9 @@
 
 import pytest
 
-from signac.core.synced_collections.collection_json import JSONDict
-from signac.core.synced_collections.synced_list import SyncedList
-from signac.core.synced_collections.utils import (
+from signac.synced_collections import SyncedList
+from signac.synced_collections.backends.collection_json import JSONDict
+from signac.synced_collections.utils import (
     AbstractTypeResolver,
     SyncedCollectionJSONEncoder,
 )

tests/test_synced_collections/test_validators.py

@@ -3,11 +3,8 @@
 # This software is licensed under the BSD 3-Clause License.
 import pytest
 
-from signac.core.synced_collections.validators import (
-    json_format_validator,
-    no_dot_in_key,
-)
-from signac.errors import InvalidKeyError, KeyTypeError
+from signac.synced_collections.errors import InvalidKeyError, KeyTypeError
+from signac.synced_collections.validators import json_format_validator, no_dot_in_key
 
 try:
     import numpy

tests/test_synced_collections/test_zarr_collection.py

@@ -6,7 +6,7 @@
 import pytest
 from synced_collection_test import SyncedDictTest, SyncedListTest
 
-from signac.core.synced_collections.collection_zarr import (
+from signac.synced_collections.backends.collection_zarr import (
     ZarrCollection,
     ZarrDict,
     ZarrList,