Merge pull request #784 from guzman-raphael/test-bucket-rules

Update changelog, version, and minio test cases
datajoint · May 15, 2020 · de192b5 · de192b5
2 parents 518b882 + f8fb9f8
commit de192b5
Show file tree

Hide file tree

Showing 14 changed files with 48 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,15 @@
 ## Release notes
 
+### 0.12.6 -- May 15, 2020
+* Add `order_by` to `dj.kill` (#668, #779) PR #775, #783
+* Add explicit S3 bucket and file storage location existence checks (#748) PR #781
+* Modify `_update` to allow nullable updates for strings/date (#664) PR #760
+* Avoid logging events on auxiliary tables (#737) PR #753
+* Add `kill_quick` and expand display to include host (#740) PR #741
+* Bugfix - pandas insert fails due to additional `index` field (#666) PR #776
+* Bugfix - `delete_external_files=True` does not remove from S3 (#686) PR #781
+* Bugfix - pandas fetch throws error when `fetch_format='frame'` PR #774
+
 ### 0.12.5 -- Feb 24, 2020
 * Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732
 * `dj.create_virtual_module` is now called `dj.VirtualModule` (#731) PR #732

diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml
@@ -19,25 +19,26 @@ services:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
     # ports:
-    #   - "80:80"
+    #   - "9000:9000"
     # volumes:
     #   - ./minio/config:/root/.minio
     #   - ./minio/data:/data
-    command: server --address ":80" /data
+    command: server --address ":9000" /data
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"]
+      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
       timeout: 5s
       retries: 60
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.5
+    image: raphaelguzman/nginx:v0.0.6
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
     - ADD_minio_TYPE=MINIO
-    - ADD_minio_ENDPOINT=minio:80
-    - ADD_minio_PREFIX=/
+    - ADD_minio_ENDPOINT=minio:9000
+    - ADD_minio_PORT=80 # allow unencrypted connections
+    - ADD_minio_PREFIX=/datajoint
     # ports:
     #   - "80:80"
     #   - "443:443"
@@ -63,7 +64,7 @@ services:
       - S3_ENDPOINT=fakeservices.datajoint.io
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
-      - S3_BUCKET=datajoint-test
+      - S3_BUCKET=datajoint.test
       - PYTHON_USER=dja
       - JUPYTER_PASSWORD=datajoint
       - DISPLAY

diff --git a/datajoint/version.py b/datajoint/version.py
@@ -1,3 +1,3 @@
-__version__ = "0.12.5"
+__version__ = "0.12.6"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters
diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst
@@ -1,3 +1,14 @@
+0.12.6 -- May 15, 2020
+----------------------
+* Add `order_by` to `dj.kill` (#668, #779) PR #775, #783
+* Add explicit S3 bucket and file storage location existence checks (#748) PR #781
+* Modify `_update` to allow nullable updates for strings/date (#664) PR #760
+* Avoid logging events on auxiliary tables (#737) PR #753
+* Add `kill_quick` and expand display to include host (#740) PR #741
+* Bugfix - pandas insert fails due to additional `index` field (#666) PR #776
+* Bugfix - `delete_external_files=True` does not remove from S3 (#686) PR #781
+* Bugfix - pandas fetch throws error when `fetch_format='frame'` PR #774
+
 0.12.5 -- Feb 24, 2020
 ----------------------
 * Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732

diff --git a/local-docker-compose.yml b/local-docker-compose.yml
@@ -20,30 +20,35 @@ services:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
     # ports:
-    #   - "80:80"
+    #   - "9000:9000"
     # To persist MinIO data and config
     # volumes:
     #   - ./minio/data:/data
     #   - ./minio/config:/root/.minio
-    command: server --address ":80" /data
+    command: server --address ":9000" /data
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"]
+      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
       timeout: 5s
       retries: 60
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.5
+    image: raphaelguzman/nginx:v0.0.6
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
     - ADD_minio_TYPE=MINIO
-    - ADD_minio_ENDPOINT=minio:80
-    - ADD_minio_PREFIX=/
+    - ADD_minio_ENDPOINT=minio:9000
+    - ADD_minio_PORT=80 # allow unencrypted connections
+    - ADD_minio_PREFIX=/datajoint
+    - ADD_browser_TYPE=MINIOADMIN
+    - ADD_browser_ENDPOINT=minio:9000
+    - ADD_browser_PORT=80 # allow unencrypted connections
     ports:
       - "80:80"
       - "443:443"
       - "3306:3306"
+      - "9000:9000"
     depends_on:
       db:
         condition: service_healthy
@@ -66,7 +71,7 @@ services:
       - S3_ENDPOINT=fakeservices.datajoint.io
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
-      - S3_BUCKET=datajoint-test
+      - S3_BUCKET=datajoint.test
       - PYTHON_USER=dja
       - JUPYTER_PASSWORD=datajoint
       - DISPLAY

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -39,9 +39,9 @@
     endpoint=environ.get('S3_ENDPOINT', 'localhost:9000'),
     access_key=environ.get('S3_ACCESS_KEY', 'datajoint'),
     secret_key=environ.get('S3_SECRET_KEY', 'datajoint'),
-    bucket=environ.get('S3_BUCKET', 'datajoint-test'))
+    bucket=environ.get('S3_BUCKET', 'datajoint.test'))
 
-S3_MIGRATE_BUCKET = [path.stem for path in Path(
+S3_MIGRATE_BUCKET = [path.name for path in Path(
         Path(__file__).resolve().parent,
         'external-legacy-data', 's3').iterdir()][0]
 
@@ -104,7 +104,7 @@
     S3_CONN_INFO['endpoint'],
     access_key=S3_CONN_INFO['access_key'],
     secret_key=S3_CONN_INFO['secret_key'],
-    secure=False,
+    secure=True,
     http_client=httpClient)
 
 

diff --git a/...BB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local → ...BB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local b/...BB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local → ...BB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local
diff --git a/...W9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal → ...W9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal b/...W9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal → ...W9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal
diff --git a/...WM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared → ...WM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared b/...WM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared → ...WM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared
diff --git a/...tKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared → ...tKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared b/...tKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared → ...tKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared
diff --git a/...VZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared → ...VZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared b/...VZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared → ...VZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared
diff --git a/...A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA → ...A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA b/...A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA → ...A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA
diff --git a/...hi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 → ...hi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 b/...hi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 → ...hi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94
diff --git a/tests/test_s3.py b/tests/test_s3.py
@@ -3,6 +3,7 @@
 from minio import Minio
 import urllib3
 import certifi
+from nose.tools import assert_true
 
 
 class TestS3:
@@ -27,7 +28,7 @@ def test_connection():
             secure=False,
             http_client=http_client)
 
-        buckets = minio_client.list_buckets()
+        assert_true(minio_client.bucket_exists(S3_CONN_INFO['bucket']))
 
     @staticmethod
     def test_connection_secure():
@@ -49,4 +50,4 @@ def test_connection_secure():
             secure=True,
             http_client=http_client)
 
-        buckets = minio_client.list_buckets()
+        assert_true(minio_client.bucket_exists(S3_CONN_INFO['bucket']))