Merge pull request #732 from guzman-raphael/schema

Rename dj.schema to dj.Schema and dj.create_virtual_module to dj.VirtualModule

CHANGELOG.md

@@ -1,5 +1,12 @@
 ## Release notes
 
+### 0.12.5 -- Feb TBA, 2020
+* Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732
+* `dj.create_virtual_module` is now called `dj.VirtualModule` (#731) PR #732
+* Bugfix - SSL `KeyError` on failed connection (#716) PR #725
+* Bugfix - Unable to run unit tests using nosetests (#723) PR #724
+* Bugfix - `suppress_errors` does not suppress loss of connection error (#720) PR #721
+
 ### 0.12.4 -- Jan 14, 2020
 * Support for simple scalar datatypes in blobs (#690) PR #709
 * Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713

LNX-docker-compose.yml

@@ -14,10 +14,10 @@ services:
       fakeservices.datajoint.io:
         condition: service_healthy
     environment:
-      - DJ_HOST=db
+      - DJ_HOST=fakeservices.datajoint.io
       - DJ_USER=root
       - DJ_PASS=simple
-      - DJ_TEST_HOST=db
+      - DJ_TEST_HOST=fakeservices.datajoint.io
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
       - S3_ENDPOINT=fakeservices.datajoint.io:9000
@@ -57,10 +57,10 @@ services:
     #   - ./mysql/data:/var/lib/mysql
   minio:
     <<: *net
+    image: minio/minio:$MINIO_VER
     environment:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
-    image: minio/minio:$MINIO_VER
     # ports:
     #   - "9000:9000"
     # volumes:
@@ -79,6 +79,7 @@ services:
       - URL=datajoint.io
       - SUBDOMAINS=fakeservices
       - MINIO_SERVER=http://minio:9000
+      - MYSQL_SERVER=db:3306
     entrypoint: /entrypoint.sh
     healthcheck:
       test: wget --quiet --tries=1 --spider https://fakeservices.datajoint.io:443/minio/health/live || exit 1
@@ -88,8 +89,10 @@ services:
     # ports:
     #   - "9000:9000"
     #   - "443:443"
+    #   - "3306:3306"
     volumes:
       - ./tests/nginx/base.conf:/base.conf
+      - ./tests/nginx/nginx.conf:/nginx.conf
       - ./tests/nginx/entrypoint.sh:/entrypoint.sh
       - ./tests/nginx/fullchain.pem:/certs/fullchain.pem
       - ./tests/nginx/privkey.pem:/certs/privkey.pem

datajoint/__init__.py

@@ -18,8 +18,8 @@
 __date__ = "February 7, 2019"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'Connection',
-           'schema', 'create_virtual_module', 'list_schemas',
-           'Table', 'FreeTable',
+           'Schema', 'schema', 'VirtualModule', 'create_virtual_module',
+           'list_schemas', 'Table', 'FreeTable',
            'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
            'Not', 'AndList', 'U', 'Diagram', 'Di', 'ERD',
            'set_password', 'kill',
@@ -29,8 +29,8 @@
 from .version import __version__
 from .settings import config
 from .connection import conn, Connection
-from .schema import Schema as schema
-from .schema import create_virtual_module, list_schemas
+from .schemas import Schema
+from .schemas import VirtualModule, list_schemas
 from .table import Table, FreeTable
 from .user_tables import Manual, Lookup, Imported, Computed, Part
 from .expression import Not, AndList, U
@@ -43,4 +43,6 @@
 from .errors import DataJointError
 from .migrate import migrate_dj011_external_blob_storage_to_dj012
 
-ERD = Di = Diagram   # Aliases for Diagram
+ERD = Di = Diagram                      # Aliases for Diagram
+schema = Schema                         # Aliases for Schema
+create_virtual_module = VirtualModule   # Aliases for VirtualModule

datajoint/migrate.py

@@ -25,7 +25,7 @@ def migrate_dj011_external_blob_storage_to_dj012(migration_schema, store):
 Proceed?
             """, default='no') == 'yes'
     if do_migration:
-        _migrate_dj011_blob(dj.schema(migration_schema), store)
+        _migrate_dj011_blob(dj.Schema(migration_schema), store)
         print('Migration completed for schema: {}, store: {}.'.format(
                 migration_schema, store))
         return

datajoint/schema.py → datajoint/schemas.py

@@ -280,8 +280,8 @@ def repl(s):
         body = '\n\n\n'.join(make_class_definition(table) for table in diagram.topological_sort())
         python_code = '\n\n\n'.join((
             '"""This module was auto-generated by datajoint from an existing schema"""',
-            "import datajoint as dj\n\nschema = dj.schema('{db}')".format(db=db),
-            '\n'.join("{module} = dj.create_virtual_module('{module}', '{schema_name}')".format(module=v, schema_name=k)
+            "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
+            '\n'.join("{module} = dj.VirtualModule('{module}', '{schema_name}')".format(module=v, schema_name=k)
                       for k, v in module_lookup.items()), body))
         if python_filename is None:
             return python_code
@@ -290,26 +290,30 @@ def repl(s):
                 f.write(python_code)
 
 
-def create_virtual_module(module_name, schema_name, *,
-                          create_schema=False, create_tables=False, connection=None, add_objects=None):
+class VirtualModule(types.ModuleType):
     """
-    Creates a python module with the given name from the name of a schema on the server and
-    automatically adds classes to it corresponding to the tables in the schema.
-    :param module_name: displayed module name
-    :param schema_name: name of the database in mysql
-    :param create_schema: if True, create the schema on the database server
-    :param create_tables: if True, module.schema can be used as the decorator for declaring new
-    :param connection: a dj.Connection object to pass into the schema
-    :param add_objects: additional objects to add to the module
-    :return: the python module containing classes from the schema object and the table classes
+    A virtual module which will contain context for schema.
     """
-    module = types.ModuleType(module_name)
-    _schema = Schema(schema_name, create_schema=create_schema, create_tables=create_tables, connection=connection)
-    if add_objects:
-        module.__dict__.update(add_objects)
-    module.__dict__['schema'] = _schema
-    _schema.spawn_missing_classes(context=module.__dict__)
-    return module
+    def __init__(self, module_name, schema_name, *, create_schema=False, 
+                    create_tables=False, connection=None, add_objects=None):
+        """
+        Creates a python module with the given name from the name of a schema on the server and
+        automatically adds classes to it corresponding to the tables in the schema.
+        :param module_name: displayed module name
+        :param schema_name: name of the database in mysql
+        :param create_schema: if True, create the schema on the database server
+        :param create_tables: if True, module.schema can be used as the decorator for declaring new
+        :param connection: a dj.Connection object to pass into the schema
+        :param add_objects: additional objects to add to the module
+        :return: the python module containing classes from the schema object and the table classes
+        """
+        super(VirtualModule, self).__init__(name=module_name)
+        _schema = Schema(schema_name, create_schema=create_schema, create_tables=create_tables,
+                            connection=connection)
+        if add_objects:
+            self.__dict__.update(add_objects)
+        self.__dict__['schema'] = _schema
+        _schema.spawn_missing_classes(context=self.__dict__)
 
 
 def list_schemas(connection=None):

datajoint/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.12.4"
+__version__ = "0.12.5"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters

docs-parts/definition/01-Creating-Schemas_lang1.rst

@@ -2,21 +2,21 @@
 .. note:: By convention, the ``datajoint`` package is imported as ``dj``.
   The documentation refers to the package as ``dj`` throughout.
 
-Create a new schema using the ``dj.schema`` function:
+Create a new schema using the ``dj.Schema`` class object:
 
 .. code-block:: python
 
     import datajoint as dj
-    schema = dj.schema('alice_experiment')
+    schema = dj.Schema('alice_experiment')
 
 This statement creates the database schema ``alice_experiment`` on the server.
 
 The returned object ``schema`` will then serve as a decorator for DataJoint classes, as described in :ref:`table`.
 
 It is a common practice to have a separate Python module for each schema.
-Therefore, each such module has only one ``dj.schema`` object defined and is usually named ``schema``.
+Therefore, each such module has only one ``dj.Schema`` object defined and is usually named ``schema``.
 
-The ``dj.schema`` constructor can take a number of optional parameters after the schema name.
+The ``dj.Schema`` constructor can take a number of optional parameters after the schema name.
 
 - ``context`` - Dictionary for looking up foreign key references.
   Defaults to ``None`` to use local context.

docs-parts/definition/02-Creating-Tables_lang1.rst

@@ -12,7 +12,7 @@ For example, the following code defines the table ``Person``:
 .. code-block:: python
 
 	import datajoint as dj
-	schema = dj.schema('alice_experiment')
+	schema = dj.Schema('alice_experiment')
 
 	@schema
 	class Person(dj.Manual):

docs-parts/definition/11-ERD_lang1.rst

@@ -4,7 +4,7 @@ To plot the ERD for an entire schema, an ERD object can be initialized with the
 .. code-block:: python
 
     import datajoint as dj
-    schema = dj.schema('my_database')
+    schema = dj.Schema('my_database')
     dj.ERD(schema).draw()
 
 or alternatively an object that has the schema object as an attribute, such as the module defining a schema:

docs-parts/existing/0-Virtual-Modules_lang1.rst

@@ -1,4 +1,4 @@
-The function ``create_virtual_module`` of the ``dj.schema`` class provides access to virtual modules.
+The class object ``VirtualModule`` of the ``dj.Schema`` class provides access to virtual modules.
 It creates a python module with the given name from the name of a schema on the server, automatically adds classes to it corresponding to the tables in the schema.
 
 The function can take several parameters:

docs-parts/existing/1-Loading-Classes_lang1.rst

@@ -89,9 +89,9 @@ the chosen database schema:
 
 .. code-block:: python
 
-    schema = dj.schema('dimitri_university')
+    schema = dj.Schema('dimitri_university')
 
-If the schema already exists, dj.schema is initialized as usual and you may plot
+If the schema already exists, dj.Schema is initialized as usual and you may plot
 the schema diagram. But instead of seeing class names, you will see the raw
 table names as they appear in the database.
 
@@ -150,13 +150,13 @@ equivalent to the Python command
     import university as uni
 
 We can mimick this import without having access to ``university.py`` using the
-``create_virtual_module`` function:
+``VirtualModule`` class object:
 
 .. code-block:: python
 
     import datajoint as dj
 
-    uni = dj.create_virtual_module('university.py', 'dimitri_university')
+    uni = dj.VirtualModule('university.py', 'dimitri_university')
 
 *Connecting dimitri@localhost:3306*
 
@@ -182,14 +182,14 @@ the table classes.
    :align: center
    :alt: query object preview
 
-``dj.create_virtual_module`` takes optional arguments.
+``dj.VirtualModule`` takes optional arguments.
 
 First, ``create_schema=False`` assures that an error is raised when the schema
 does not already exist. Set it to ``True`` if you want to create an empty schema.
 
 .. code-block:: python
 
-    dj.create_virtual_module('what', 'nonexistent')
+    dj.VirtualModule('what', 'nonexistent')
 
 .. code-block:: python
 
@@ -215,7 +215,7 @@ decorator for declaring new tables:
 
 .. code-block:: python
 
-    uni = dj.create_virtual_module('university.py', 'dimitri_university', create_tables=True)
+    uni = dj.VirtualModule('university.py', 'dimitri_university', create_tables=True)
 
 .. code-block:: python
 

docs-parts/intro/Releases_lang1.rst

@@ -1,3 +1,10 @@
+0.12.5 -- Feb TBA, 2020
+* Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732
+* `dj.create_virtual_module` is now called `dj.VirtualModule` (#731) PR #732
+* Bugfix - SSL `KeyError` on failed connection (#716) PR #725
+* Bugfix - Unable to run unit tests using nosetests (#723) PR #724
+* Bugfix - `suppress_errors` does not suppress loss of connection error (#720) PR #721
+
 0.12.4 -- Jan 14, 2020
 * Support for simple scalar datatypes in blobs (#690) PR #709
 * Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713

local-docker-compose.yml

@@ -14,10 +14,10 @@ services:
       fakeservices.datajoint.io:
         condition: service_healthy
     environment:
-      - DJ_HOST=db
+      - DJ_HOST=fakeservices.datajoint.io
       - DJ_USER=root
       - DJ_PASS=simple
-      - DJ_TEST_HOST=db
+      - DJ_TEST_HOST=fakeservices.datajoint.io
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
       # If running tests locally, make sure to add entry in /etc/hosts for 127.0.0.1 fakeservices.datajoint.io
@@ -35,7 +35,7 @@ services:
         pip install --user nose nose-cov coveralls ptvsd .;
         pip freeze | grep datajoint;
         ## You may run the below tests once sh'ed into container i.e. docker exec -it datajoint-python_app_1 sh
-        # nosetests -vsw tests --with-coverage --cover-package=datajoint; #run all tests
+        # nosetests -vsw tests; #run all tests
         # nosetests -vs --tests=tests.test_external_class:test_insert_and_fetch; #run specific basic test
         # nosetests -vs --tests=tests.test_fetch:TestFetch.test_getattribute_for_fetch1; #run specific Class test
         ## Interactive Jupyter Notebook environment
@@ -61,17 +61,19 @@ services:
     image: datajoint/mysql:$MYSQL_VER
     environment:
       - MYSQL_ROOT_PASSWORD=simple
-    ports:
-      - "3306:3306"
+    # ports:
+    #   - "3306:3306"
     # To persist MySQL data
     # volumes:
     #   - ./mysql/data:/var/lib/mysql
   minio:
     <<: *net
+    image: minio/minio:$MINIO_VER
     environment:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
-    image: minio/minio:$MINIO_VER
+    # ports:
+    #   - "9000:9000"
     # To persist MinIO data and config
     # volumes:
     #   - ./minio/data:/data
@@ -89,6 +91,7 @@ services:
       - URL=datajoint.io
       - SUBDOMAINS=fakeservices
       - MINIO_SERVER=http://minio:9000
+      - MYSQL_SERVER=db:3306
     entrypoint: /entrypoint.sh
     healthcheck:
       test: wget --quiet --tries=1 --spider https://fakeservices.datajoint.io:443/minio/health/live || exit 1
@@ -98,8 +101,10 @@ services:
     ports:
       - "9000:9000"
       - "443:443"
+      - "3306:3306"
     volumes:
       - ./tests/nginx/base.conf:/base.conf
+      - ./tests/nginx/nginx.conf:/nginx.conf
       - ./tests/nginx/entrypoint.sh:/entrypoint.sh
       - ./tests/nginx/fullchain.pem:/certs/fullchain.pem
       - ./tests/nginx/privkey.pem:/certs/privkey.pem

tests/nginx/entrypoint.sh

@@ -2,4 +2,5 @@
 sed "s|{{SUBDOMAINS}}|${SUBDOMAINS}|g" /base.conf | tee /etc/nginx/conf.d/base.conf
 sed -i "s|{{URL}}|${URL}|g" /etc/nginx/conf.d/base.conf
 sed -i "s|{{MINIO_SERVER}}|${MINIO_SERVER}|g" /etc/nginx/conf.d/base.conf
+sed "s|{{MYSQL_SERVER}}|${MYSQL_SERVER}|g" /nginx.conf | tee /etc/nginx/nginx.conf
 nginx -g "daemon off;"

tests/nginx/nginx.conf

@@ -0,0 +1,42 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+stream {
+    upstream target_db {
+    server {{MYSQL_SERVER}};
+  }
+
+  server {
+    listen 3306;
+    proxy_pass target_db;
+  }
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}

tests/schema.py

@@ -8,7 +8,7 @@
 import inspect
 from . import PREFIX, CONN_INFO
 
-schema = dj.schema(PREFIX + '_test1', connection=dj.conn(**CONN_INFO))
+schema = dj.Schema(PREFIX + '_test1', connection=dj.conn(**CONN_INFO))
 
 
 @schema