diff --git a/CHANGELOG.md b/CHANGELOG.md index 325099fb4..5f45b082d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## Release notes +### 0.12.6 -- May 15, 2020 +* Add `order_by` to `dj.kill` (#668, #779) PR #775, #783 +* Add explicit S3 bucket and file storage location existence checks (#748) PR #781 +* Modify `_update` to allow nullable updates for strings/date (#664) PR #760 +* Avoid logging events on auxiliary tables (#737) PR #753 +* Add `kill_quick` and expand display to include host (#740) PR #741 +* Bugfix - pandas insert fails due to additional `index` field (#666) PR #776 +* Bugfix - `delete_external_files=True` does not remove from S3 (#686) PR #781 +* Bugfix - pandas fetch throws error when `fetch_format='frame'` PR #774 + ### 0.12.5 -- Feb 24, 2020 * Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732 * `dj.create_virtual_module` is now called `dj.VirtualModule` (#731) PR #732 diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index 30a1ef4b1..6ae2dabb4 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -19,25 +19,26 @@ services: - MINIO_ACCESS_KEY=datajoint - MINIO_SECRET_KEY=datajoint # ports: - # - "80:80" + # - "9000:9000" # volumes: # - ./minio/config:/root/.minio # - ./minio/data:/data - command: server --address ":80" /data + command: server --address ":9000" /data healthcheck: - test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"] + test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"] timeout: 5s retries: 60 interval: 1s fakeservices.datajoint.io: <<: *net - image: raphaelguzman/nginx:v0.0.5 + image: raphaelguzman/nginx:v0.0.6 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 - ADD_minio_TYPE=MINIO - - ADD_minio_ENDPOINT=minio:80 - - ADD_minio_PREFIX=/ + - ADD_minio_ENDPOINT=minio:9000 + - ADD_minio_PORT=80 # allow unencrypted connections + - ADD_minio_PREFIX=/datajoint # ports: # - "80:80" # - "443:443" @@ -63,7 +64,7 @@ services: - S3_ENDPOINT=fakeservices.datajoint.io - S3_ACCESS_KEY=datajoint - S3_SECRET_KEY=datajoint - - S3_BUCKET=datajoint-test + - S3_BUCKET=datajoint.test - PYTHON_USER=dja - JUPYTER_PASSWORD=datajoint - DISPLAY diff --git a/datajoint/version.py b/datajoint/version.py index b8ab52e90..210836658 100644 --- a/datajoint/version.py +++ b/datajoint/version.py @@ -1,3 +1,3 @@ -__version__ = "0.12.5" +__version__ = "0.12.6" assert len(__version__) <= 10 # The log table limits version to the 10 characters diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index 358f60d5b..c91e2e554 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,3 +1,14 @@ +0.12.6 -- May 15, 2020 +---------------------- +* Add `order_by` to `dj.kill` (#668, #779) PR #775, #783 +* Add explicit S3 bucket and file storage location existence checks (#748) PR #781 +* Modify `_update` to allow nullable updates for strings/date (#664) PR #760 +* Avoid logging events on auxiliary tables (#737) PR #753 +* Add `kill_quick` and expand display to include host (#740) PR #741 +* Bugfix - pandas insert fails due to additional `index` field (#666) PR #776 +* Bugfix - `delete_external_files=True` does not remove from S3 (#686) PR #781 +* Bugfix - pandas fetch throws error when `fetch_format='frame'` PR #774 + 0.12.5 -- Feb 24, 2020 ---------------------- * Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732 diff --git a/local-docker-compose.yml b/local-docker-compose.yml index b0b976c41..9648df0ce 100644 --- a/local-docker-compose.yml +++ b/local-docker-compose.yml @@ -20,30 +20,35 @@ services: - MINIO_ACCESS_KEY=datajoint - MINIO_SECRET_KEY=datajoint # ports: - # - "80:80" + # - "9000:9000" # To persist MinIO data and config # volumes: # - ./minio/data:/data # - ./minio/config:/root/.minio - command: server --address ":80" /data + command: server --address ":9000" /data healthcheck: - test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"] + test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"] timeout: 5s retries: 60 interval: 1s fakeservices.datajoint.io: <<: *net - image: raphaelguzman/nginx:v0.0.5 + image: raphaelguzman/nginx:v0.0.6 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 - ADD_minio_TYPE=MINIO - - ADD_minio_ENDPOINT=minio:80 - - ADD_minio_PREFIX=/ + - ADD_minio_ENDPOINT=minio:9000 + - ADD_minio_PORT=80 # allow unencrypted connections + - ADD_minio_PREFIX=/datajoint + - ADD_browser_TYPE=MINIOADMIN + - ADD_browser_ENDPOINT=minio:9000 + - ADD_browser_PORT=80 # allow unencrypted connections ports: - "80:80" - "443:443" - "3306:3306" + - "9000:9000" depends_on: db: condition: service_healthy @@ -66,7 +71,7 @@ services: - S3_ENDPOINT=fakeservices.datajoint.io - S3_ACCESS_KEY=datajoint - S3_SECRET_KEY=datajoint - - S3_BUCKET=datajoint-test + - S3_BUCKET=datajoint.test - PYTHON_USER=dja - JUPYTER_PASSWORD=datajoint - DISPLAY diff --git a/tests/__init__.py b/tests/__init__.py index e5a4f835e..90e8b8922 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -39,9 +39,9 @@ endpoint=environ.get('S3_ENDPOINT', 'localhost:9000'), access_key=environ.get('S3_ACCESS_KEY', 'datajoint'), secret_key=environ.get('S3_SECRET_KEY', 'datajoint'), - bucket=environ.get('S3_BUCKET', 'datajoint-test')) + bucket=environ.get('S3_BUCKET', 'datajoint.test')) -S3_MIGRATE_BUCKET = [path.stem for path in Path( +S3_MIGRATE_BUCKET = [path.name for path in Path( Path(__file__).resolve().parent, 'external-legacy-data', 's3').iterdir()][0] @@ -104,7 +104,7 @@ S3_CONN_INFO['endpoint'], access_key=S3_CONN_INFO['access_key'], secret_key=S3_CONN_INFO['secret_key'], - secure=False, + secure=True, http_client=httpClient) diff --git a/tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local b/tests/external-legacy-data/file/temp/datajoint.migrate/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local similarity index 100% rename from tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local rename to tests/external-legacy-data/file/temp/datajoint.migrate/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local diff --git a/tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal b/tests/external-legacy-data/file/temp/datajoint.migrate/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal similarity index 100% rename from tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal rename to tests/external-legacy-data/file/temp/datajoint.migrate/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal diff --git a/tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared b/tests/external-legacy-data/s3/datajoint.migrate/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared similarity index 100% rename from tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared rename to tests/external-legacy-data/s3/datajoint.migrate/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared diff --git a/tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared b/tests/external-legacy-data/s3/datajoint.migrate/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared similarity index 100% rename from tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared rename to tests/external-legacy-data/s3/datajoint.migrate/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared diff --git a/tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared b/tests/external-legacy-data/s3/datajoint.migrate/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared similarity index 100% rename from tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared rename to tests/external-legacy-data/s3/datajoint.migrate/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared diff --git a/tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA b/tests/external-legacy-data/s3/datajoint.migrate/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA similarity index 100% rename from tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA rename to tests/external-legacy-data/s3/datajoint.migrate/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA diff --git a/tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 b/tests/external-legacy-data/s3/datajoint.migrate/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 similarity index 100% rename from tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 rename to tests/external-legacy-data/s3/datajoint.migrate/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 diff --git a/tests/test_s3.py b/tests/test_s3.py index 4036f5d81..73ae89f9e 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -3,6 +3,7 @@ from minio import Minio import urllib3 import certifi +from nose.tools import assert_true class TestS3: @@ -27,7 +28,7 @@ def test_connection(): secure=False, http_client=http_client) - buckets = minio_client.list_buckets() + assert_true(minio_client.bucket_exists(S3_CONN_INFO['bucket'])) @staticmethod def test_connection_secure(): @@ -49,4 +50,4 @@ def test_connection_secure(): secure=True, http_client=http_client) - buckets = minio_client.list_buckets() + assert_true(minio_client.bucket_exists(S3_CONN_INFO['bucket']))