From 4aa011ee4b28929f7dac410e5d6b8d1111e80115 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 19 Nov 2019 15:22:05 -0600 Subject: [PATCH 01/12] Fix #698 --- datajoint/table.py | 11 +++-------- datajoint/user_tables.py | 4 ++-- tests/schema.py | 3 +++ tests/test_relation.py | 6 ++++++ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/datajoint/table.py b/datajoint/table.py index 0c093f0cd..4a44f48d5 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -45,14 +45,9 @@ def heading(self): """ if self._heading is None: self._heading = Heading() # instance-level heading - if not self._heading: # lazy loading of heading - if self.connection is None: - raise DataJointError( - 'DataJoint class is missing a database connection. ' - 'Missing schema decorator on the class? (e.g. @schema)') - else: - self._heading.init_from_database( - self.connection, self.database, self.table_name, self.declaration_context) + if not self._heading and self.connection is not None: # lazy loading of heading + self._heading.init_from_database( + self.connection, self.database, self.table_name, self.declaration_context) return self._heading def declare(self, context=None): diff --git a/datajoint/user_tables.py b/datajoint/user_tables.py index 216e4b37c..3942264b5 100644 --- a/datajoint/user_tables.py +++ b/datajoint/user_tables.py @@ -13,7 +13,7 @@ # attributes that trigger instantiation of user classes supported_class_attrs = { 'key_source', 'describe', 'alter', 'heading', 'populate', 'progress', 'primary_key', 'proj', 'aggr', - 'fetch', 'fetch1','head', 'tail', + 'fetch', 'fetch1', 'head', 'tail', 'insert', 'insert1', 'drop', 'drop_quick', 'delete', 'delete_quick'} @@ -92,7 +92,7 @@ def table_name(cls): @ClassProperty def full_table_name(cls): - if cls not in {Manual, Imported, Lookup, Computed, Part}: + if cls not in {Manual, Imported, Lookup, Computed, Part, UserTable}: if cls.database is None: raise DataJointError('Class %s is not properly declared (schema decorator not applied?)' % cls.__name__) return r"`{0:s}`.`{1:s}`".format(cls.database, cls.table_name) diff --git a/tests/schema.py b/tests/schema.py index 960ed3dda..7848274be 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -13,6 +13,9 @@ @schema class TTest(dj.Lookup): + """ + doc string + """ definition = """ key : int # key --- diff --git a/tests/test_relation.py b/tests/test_relation.py index eecce31ed..734bdc03a 100644 --- a/tests/test_relation.py +++ b/tests/test_relation.py @@ -36,6 +36,12 @@ def setup_class(cls): cls.img = schema.Image() cls.trash = schema.UberTrash() + def test_class_help(self): + help(schema.TTest) + + def test_instance_help(self): + help(schema.TTest()) + def test_contents(self): """ test the ability of tables to self-populate using the contents property From 41acaa32a51514a49a32eb515a6c10fb66ca5328 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 19 Nov 2019 16:09:01 -0600 Subject: [PATCH 02/12] fix #699 -- add table definition to doc string --- datajoint/schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datajoint/schema.py b/datajoint/schema.py index c573b81de..c6e777d63 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -191,6 +191,10 @@ def process_table_class(self, table_class, context, assert_declared=False): instance.declare(context) is_declared = is_declared or instance.is_declared + # add table definition to the doc string + if isinstance(table_class.definition, str): + table_class.__doc__ = (table_class.__doc__ or "") + "\n\nTable definition:\n\n" + table_class.definition + # fill values in Lookup tables from their contents property if isinstance(instance, Lookup) and hasattr(instance, 'contents') and is_declared: contents = list(instance.contents) From 503a170236c37558745708912f91b9ec13ba33b8 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 19 Nov 2019 16:24:55 -0600 Subject: [PATCH 03/12] Table doc strings now display reverse-engineered table declarations --- datajoint/heading.py | 3 ++- datajoint/schema.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datajoint/heading.py b/datajoint/heading.py index 60f636b02..a025fb130 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -247,7 +247,8 @@ def init_from_database(self, conn, database, table_name, context): category = next(c for c in SPECIAL_TYPES if TYPE_PATTERN[c].match(attr['type'])) except StopIteration: if attr['type'].startswith('external'): - raise DataJointError('Legacy datatype `{type}`.'.format(**attr)) from None + raise DataJointError('Legacy datatype `{type}`. ' + 'Migrate your external stores to datajoint 0.12'.format(**attr)) from None raise DataJointError('Unknown attribute type `{type}`'.format(**attr)) from None if category == 'FILEPATH' and not _support_filepath_types(): raise DataJointError(""" diff --git a/datajoint/schema.py b/datajoint/schema.py index c6e777d63..eb9dfba52 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -193,7 +193,8 @@ def process_table_class(self, table_class, context, assert_declared=False): # add table definition to the doc string if isinstance(table_class.definition, str): - table_class.__doc__ = (table_class.__doc__ or "") + "\n\nTable definition:\n\n" + table_class.definition + table_class.__doc__ = ((table_class.__doc__ or "") + "\n\nTable definition:\n" + + table_class.describe(printout=False)) # fill values in Lookup tables from their contents property if isinstance(instance, Lookup) and hasattr(instance, 'contents') and is_declared: From 9264e4d192a6b7a01b9b93a6e85d5cec205e0489 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 19 Nov 2019 16:37:33 -0600 Subject: [PATCH 04/12] update error message for un-upgraded external stores. --- datajoint/heading.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datajoint/heading.py b/datajoint/heading.py index a025fb130..217657423 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -247,8 +247,10 @@ def init_from_database(self, conn, database, table_name, context): category = next(c for c in SPECIAL_TYPES if TYPE_PATTERN[c].match(attr['type'])) except StopIteration: if attr['type'].startswith('external'): - raise DataJointError('Legacy datatype `{type}`. ' - 'Migrate your external stores to datajoint 0.12'.format(**attr)) from None + url = "https://docs.datajoint.io/python/admin/5-blob-config.html" \ + "#migration-between-datajoint-v0-11-and-v0-12" + raise DataJointError('Legacy datatype `{type}`. Migrate your external stores to ' + 'datajoint 0.12: {url}'.format(url=url, **attr)) from None raise DataJointError('Unknown attribute type `{type}`'.format(**attr)) from None if category == 'FILEPATH' and not _support_filepath_types(): raise DataJointError(""" From ae53cddc7080343d74d90060fcb8784713cc2a43 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 19 Nov 2019 16:54:53 -0600 Subject: [PATCH 05/12] improve table definition in the doc string --- datajoint/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/schema.py b/datajoint/schema.py index eb9dfba52..75b24c489 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -194,7 +194,7 @@ def process_table_class(self, table_class, context, assert_declared=False): # add table definition to the doc string if isinstance(table_class.definition, str): table_class.__doc__ = ((table_class.__doc__ or "") + "\n\nTable definition:\n" - + table_class.describe(printout=False)) + + table_class.describe(printout=False, context=context)) # fill values in Lookup tables from their contents property if isinstance(instance, Lookup) and hasattr(instance, 'contents') and is_declared: From 4533cea39c333d829bebd4645cba7fa21ee17e48 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 19 Nov 2019 16:56:02 -0600 Subject: [PATCH 06/12] minor improvement in display of table doc strings --- datajoint/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/schema.py b/datajoint/schema.py index 75b24c489..026023d07 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -193,7 +193,7 @@ def process_table_class(self, table_class, context, assert_declared=False): # add table definition to the doc string if isinstance(table_class.definition, str): - table_class.__doc__ = ((table_class.__doc__ or "") + "\n\nTable definition:\n" + table_class.__doc__ = ((table_class.__doc__ or "") + "\nTable definition:\n\n" + table_class.describe(printout=False, context=context)) # fill values in Lookup tables from their contents property From f8c3668d388b2e6c0718f7271baab03cd2580339 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Wed, 20 Nov 2019 10:44:57 -0600 Subject: [PATCH 07/12] replace .describe() with .definition to augment the table docstring --- datajoint/schema.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datajoint/schema.py b/datajoint/schema.py index 026023d07..55712f7aa 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -193,8 +193,7 @@ def process_table_class(self, table_class, context, assert_declared=False): # add table definition to the doc string if isinstance(table_class.definition, str): - table_class.__doc__ = ((table_class.__doc__ or "") + "\nTable definition:\n\n" - + table_class.describe(printout=False, context=context)) + table_class.__doc__ = (table_class.__doc__ or "") + "\nTable definition:\n\n" + table_class.definition # fill values in Lookup tables from their contents property if isinstance(instance, Lookup) and hasattr(instance, 'contents') and is_declared: From a6614585c596ddccf90d7b18b9bb709b7601c3ec Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 21 Nov 2019 11:48:44 -0600 Subject: [PATCH 08/12] improve unit test for definition in docstring --- tests/test_declare.py | 14 ++++++++++++++ tests/test_relation.py | 6 ------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_declare.py b/tests/test_declare.py index 3a734cd1d..62bd55cba 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -22,6 +22,20 @@ def test_schema_decorator(): assert_true(issubclass(Subject, dj.Lookup)) assert_true(not issubclass(Subject, dj.Part)) + @staticmethod + def test_class_help(): + help(TTest) + help(TTest2) + assert_true(TTest.definition in TTest.__doc__) + assert_true(TTest.definition in TTest2.__doc__) + + @staticmethod + def test_instance_help(): + help(TTest()) + help(TTest2()) + assert_true(TTest().definition in TTest().__doc__) + assert_true(TTest2().definition in TTest2().__doc__) + @staticmethod def test_describe(): """real_definition should match original definition""" diff --git a/tests/test_relation.py b/tests/test_relation.py index 734bdc03a..eecce31ed 100644 --- a/tests/test_relation.py +++ b/tests/test_relation.py @@ -36,12 +36,6 @@ def setup_class(cls): cls.img = schema.Image() cls.trash = schema.UberTrash() - def test_class_help(self): - help(schema.TTest) - - def test_instance_help(self): - help(schema.TTest()) - def test_contents(self): """ test the ability of tables to self-populate using the contents property From b28818dae73e96c6d8b0399b129075e8e6b51d19 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 21 Nov 2019 13:23:52 -0600 Subject: [PATCH 09/12] minor --- datajoint/table.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datajoint/table.py b/datajoint/table.py index 4a44f48d5..b7d4ced6e 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -406,7 +406,7 @@ def delete(self, verbose=True): print('About to delete:') if not already_in_transaction: - self.connection.start_transaction() + conn.start_transaction() total = 0 try: for name, table in reversed(list(delete_list.items())): @@ -418,25 +418,25 @@ def delete(self, verbose=True): except: # Delete failed, perhaps due to insufficient privileges. Cancel transaction. if not already_in_transaction: - self.connection.cancel_transaction() + conn.cancel_transaction() raise else: assert not (already_in_transaction and safe) if not total: print('Nothing to delete') if not already_in_transaction: - self.connection.cancel_transaction() + conn.cancel_transaction() else: if already_in_transaction: if verbose: print('The delete is pending within the ongoing transaction.') else: if not safe or user_choice("Proceed?", default='no') == 'yes': - self.connection.commit_transaction() + conn.commit_transaction() if verbose or safe: print('Committed.') else: - self.connection.cancel_transaction() + conn.cancel_transaction() if verbose or safe: print('Cancelled deletes.') From b783a9225704bd58d527d1a3c4b79ccc22357ddb Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 22 Nov 2019 14:23:13 -0600 Subject: [PATCH 10/12] update CHANGELOG for version 0.12.3 --- CHANGELOG.md | 5 +++++ datajoint/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec9152c5..27dd1d75d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## Release notes +## 0.12.3 -- Nov 22, 2019 +* Bugfix #675 (PR #705) networkx 2.4+ is now supported +* Bugfix #698 and #699 (PR #706) display table definition in doc string and help +* Bugfix #701 (PR #702) job reservation works with native python datatype support disabled + ### 0.12.2 -- Nov 11, 2019 * Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) * Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) diff --git a/datajoint/version.py b/datajoint/version.py index ff1ce7e17..18c950f21 100644 --- a/datajoint/version.py +++ b/datajoint/version.py @@ -1,3 +1,3 @@ -__version__ = "0.12.2" +__version__ = "0.12.3" assert len(__version__) <= 10 # The log table limits version to the 10 characters From fe461a2373cfb181066af886627af40a40aac76d Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 22 Nov 2019 14:40:45 -0600 Subject: [PATCH 11/12] update docs for release 0.12.3 --- datajoint/blob.py | 3 +++ docs-parts/intro/Releases_lang1.rst | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index 2fb5b1088..c678d7de5 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -253,6 +253,9 @@ def pack_recarray(self, array): def read_sparse_array(self): raise DataJointError('datajoint-python does not yet support sparse arrays. Issue (#590)') + def read_scalar(selfs): + + def read_decimal(self): return Decimal(self.read_string()) diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index afd2fa9ec..3e6f10782 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,4 +1,11 @@ -0.12.1 -- Nov 11, 2019 +0.12.3 -- Nov 22, 2019 +---------------------- +* Bugfix - networkx 2.4 causes error in diagrams (#675) PR #705 +* Bugfix - include definition in doc string and help (#698, #699) PR #706 +* Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 + + +0.12.2 -- Nov 11, 2019 ------------------------- * Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) * Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) From 04808ceb81565da041688fa3455a897093bc70c5 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 22 Nov 2019 15:45:07 -0600 Subject: [PATCH 12/12] blob now accepts native complex scalars --- datajoint/blob.py | 9 ++------- tests/test_blob.py | 3 +++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index c678d7de5..390ef04bd 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -152,10 +152,8 @@ def pack_blob(self, obj): return self.pack_array(np.array(obj)) if isinstance(obj, (bool, np.bool, np.bool_)): return self.pack_array(np.array(obj)) - if isinstance(obj, float): - return self.pack_array(np.array(obj, dtype=np.float64)) - if isinstance(obj, int): - return self.pack_array(np.array(obj, dtype=np.int64)) + if isinstance(obj, (float, int, complex)): + return self.pack_array(np.array(obj)) if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)): return self.pack_datetime(obj) if isinstance(obj, Decimal): @@ -253,9 +251,6 @@ def pack_recarray(self, array): def read_sparse_array(self): raise DataJointError('datajoint-python does not yet support sparse arrays. Issue (#590)') - def read_scalar(selfs): - - def read_decimal(self): return Decimal(self.read_string()) diff --git a/tests/test_blob.py b/tests/test_blob.py index 3549dd476..4520a83c0 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -23,6 +23,9 @@ def test_pack(): x = np.random.randn(10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") + x = 7j + assert_equal(x, unpack(pack(x)), "Complex scalar does not match") + x = np.float32(np.random.randn(3, 4, 5)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")