Merge branch 'master' into nested-gs

.github/workflows/deploy.yml

@@ -0,0 +1,49 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - master
+
+env:
+  DEPLOYMENT_REPOSITORY_ID: gs-nexus
+  DEPLOYMENT_REPOSITORY: gs-nexus::default::https://repo.glencoesoftware.com/repository/jzarr-snapshots/
+  NEXUS_USERNAME: ${{ secrets.PYPI_SERVER_USERNAME }}
+  NEXUS_PASSWORD: ${{ secrets.PYPI_SERVER_PASSWORD }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Install Blosc native library
+        run: sudo apt-get install -y libblosc1
+
+      - name: Set up latest Java 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: '8'
+          java-package: 'jdk'
+          architecture: 'x64'
+
+      - name: Set up Maven deployment environment
+        run: |
+          touch maven.properties
+          mkdir -p ${HOME}/.m2/
+          envsubst < settings.xml > ${HOME}/.m2/settings.xml
+
+      - name: Run tests
+        run: mvn -B test
+
+      - name: Deploy
+        run: mvn -B -DaltDeploymentRepository=${DEPLOYMENT_REPOSITORY} deploy
+
+      - name: Archive build artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: artifacts
+          path: artifacts/*.jar

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.bc.zarr</groupId>
     <artifactId>jzarr</artifactId>
-    <version>0.3.99-SNAPSHOT</version>
+    <version>0.3.3-gs-SNAPSHOT</version>
 
     <properties>
         <!-- needed in test scope to show examples -->

settings.xml

@@ -0,0 +1,9 @@
+<settings>
+    <servers>
+        <server>
+            <id>${DEPLOYMENT_REPOSITORY_ID}</id>
+            <username>${NEXUS_USERNAME}</username>
+            <password>${NEXUS_PASSWORD}</password>
+        </server>
+    </servers>
+</settings>

src/main/java/com/bc/zarr/Compressor.java

@@ -43,8 +43,10 @@ public abstract class Compressor {
     void passThrough(InputStream is, OutputStream os) throws IOException {
         final byte[] bytes = new byte[65536];
         int read = is.read(bytes);
-        while (read > 0) {
-            os.write(bytes, 0, read);
+        while (read >= 0) {
+            if (read > 0) {
+                os.write(bytes, 0, read);
+            }
             read = is.read(bytes);
         }
     }

src/main/java/com/bc/zarr/CompressorFactory.java

@@ -27,14 +27,19 @@
 package com.bc.zarr;
 
 import com.sun.jna.ptr.NativeLongByReference;
-import org.blosc.*;
+import org.blosc.BufferSizes;
+import org.blosc.IBloscDll;
+import org.blosc.JBlosc;
 
 import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.zip.Deflater;
@@ -58,10 +63,7 @@ public static Map<String, Object> getDefaultCompressorProperties() {
 
         /* blosc defaults */
         map.put("id", "blosc");
-        map.put("cname", "lz4");
-        map.put("clevel", 5);
-        map.put("blocksize", 0);
-        map.put("shuffle", 1);
+        map.putAll(BloscCompressor.defaultProperties);
 
         return map;
     }
@@ -206,28 +208,52 @@ public void uncompress(InputStream is, OutputStream os) throws IOException {
         }
     }
 
-    private static class BloscCompressor extends Compressor {
+    static class BloscCompressor extends Compressor {
+
+        final static int AUTOSHUFFLE = -1;
+        final static int NOSHUFFLE = 0;
+        final static int BYTESHUFFLE = 1;
+        final static int BITSHUFFLE = 2;
+
+        public final static String keyCname = "cname";
+        public final static String defaultCname = "lz4";
+        public final static String keyClevel = "clevel";
+        public final static int defaultCLevel = 5;
+        public final static String keyShuffle = "shuffle";
+        public final static int defaultShuffle = BYTESHUFFLE;
+        public final static String keyBlocksize = "blocksize";
+        public final static int defaultBlocksize = 0;
+        public final static int[] supportedShuffle = new int[]{/*AUTOSHUFFLE, */NOSHUFFLE, BYTESHUFFLE, BITSHUFFLE};
+        public final static String[] supportedCnames = new String[]{"zstd", "blosclz", defaultCname, "lz4hc", "zlib"/*, "snappy"*/};
+
+        public final static Map<String, Object> defaultProperties = Collections
+                .unmodifiableMap(new HashMap<String, Object>() {{
+                    put(keyCname, defaultCname);
+                    put(keyClevel, defaultCLevel);
+                    put(keyShuffle, defaultShuffle);
+                    put(keyBlocksize, defaultBlocksize);
+                }});
+
         private final int clevel;
         private final int blocksize;
         private final int shuffle;
         private final String cname;
 
         private BloscCompressor(Map<String, Object> map) {
-            final Object cnameObj = map.get("cname");
+            final Object cnameObj = map.get(keyCname);
             if (cnameObj == null) {
-                cname = "lz4"; //default value
+                cname = defaultCname;
             } else {
                 cname = (String) cnameObj;
             }
-            final String[] supportedNames = {"zstd", "blosclz", "lz4", "lz4hc", "zlib", "snappy"};
-            if (Arrays.stream(supportedNames).noneMatch(cname::equals)) {
+            if (Arrays.stream(supportedCnames).noneMatch(cname::equals)) {
                 throw new IllegalArgumentException(
-                        "blosc: compressor not supported: '" + cname + "'; expected one of " + Arrays.toString(supportedNames));
+                        "blosc: compressor not supported: '" + cname + "'; expected one of " + Arrays.toString(supportedCnames));
             }
 
-            final Object clevelObj = map.get("clevel");
+            final Object clevelObj = map.get(keyClevel);
             if (clevelObj == null) {
-                clevel = 5; //default value
+                clevel = defaultCLevel;
             } else if (clevelObj instanceof String) {
                 clevel = Integer.parseInt((String) clevelObj);
             } else {
@@ -237,29 +263,23 @@ private BloscCompressor(Map<String, Object> map) {
                 throw new IllegalArgumentException("blosc: clevel parameter must be between 0 and 9 but was: " + clevel);
             }
 
-            final int AUTOSHUFFLE = -1;
-            final int NOSHUFFLE = 0;
-            final int BYTESHUFFLE = 1;
-            final int BITSHUFFLE = 2;
-
-            final Object shuffleObj = map.get("shuffle");
+            final Object shuffleObj = map.get(keyShuffle);
             if (shuffleObj == null) {
-                this.shuffle = BYTESHUFFLE; //default value
+                this.shuffle = defaultShuffle;
             } else if (shuffleObj instanceof String) {
                 this.shuffle = Integer.parseInt((String) shuffleObj);
             } else {
                 this.shuffle = ((Number) shuffleObj).intValue();
             }
-            final int[] supportedShuffle = new int[]{AUTOSHUFFLE, NOSHUFFLE, BYTESHUFFLE, BITSHUFFLE};
-            final String[] supportedShuffleNames = new String[]{"-1 (AUTOSHUFFLE)", "0 (NOSHUFFLE)", "1 (BYTESHUFFLE)", "2 (BITSHUFFLE)"};
+            final String[] supportedShuffleNames = new String[]{/*"-1 (AUTOSHUFFLE)", */"0 (NOSHUFFLE)", "1 (BYTESHUFFLE)", "2 (BITSHUFFLE)"};
             if (Arrays.stream(supportedShuffle).noneMatch(value -> value == shuffle)) {
                 throw new IllegalArgumentException(
                         "blosc: shuffle type not supported: '" + shuffle + "'; expected one of " + Arrays.toString(supportedShuffleNames));
             }
 
-            final Object blocksizeObj = map.get("blocksize");
+            final Object blocksizeObj = map.get(keyBlocksize);
             if (blocksizeObj == null) {
-                this.blocksize = 0; //default value
+                this.blocksize = defaultBlocksize;
             } else if (blocksizeObj instanceof String) {
                 this.blocksize = Integer.parseInt((String) blocksizeObj);
             } else {
@@ -312,21 +332,17 @@ public void compress(InputStream is, OutputStream os) throws IOException {
 
         @Override
         public void uncompress(InputStream is, OutputStream os) throws IOException {
-            while (is.available() >= 16) {
-                byte[] header = new byte[16];
-                is.read(header);
-
-                BufferSizes bs = cbufferSizes(ByteBuffer.wrap(header));
-                int chunkSize = (int) bs.getCbytes();
-
-                byte[] inBytes = Arrays.copyOf(header, header.length + chunkSize);
-                is.read(inBytes, header.length, chunkSize);
-
-                ByteBuffer outBuffer = ByteBuffer.allocate((int) bs.getNbytes());
-                JBlosc.decompressCtx(ByteBuffer.wrap(inBytes), outBuffer, outBuffer.limit(), 1);
-
-                os.write(outBuffer.array());
-            }
+            final DataInput di = new DataInputStream(is);
+            byte[] header = new byte[JBlosc.OVERHEAD];
+            di.readFully(header);
+            BufferSizes bs = cbufferSizes(ByteBuffer.wrap(header));
+            int compressedSize = (int) bs.getCbytes();
+            int uncompressedSize = (int) bs.getNbytes();
+            byte[] inBytes = Arrays.copyOf(header, compressedSize);
+            di.readFully(inBytes, header.length, compressedSize - header.length);
+            ByteBuffer outBuffer = ByteBuffer.allocate(uncompressedSize);
+            JBlosc.decompressCtx(ByteBuffer.wrap(inBytes), outBuffer, outBuffer.limit(), 1);
+            os.write(outBuffer.array());
         }
 
         private BufferSizes cbufferSizes(ByteBuffer cbuffer) {

src/main/java/com/bc/zarr/ZarrArray.java

@@ -226,6 +226,10 @@ public static ZarrArray create(ZarrPath relativePath, Store store, ArrayParams a
         return zarrArray;
     }
 
+    public Compressor getCompressor() {
+        return _compressor;
+    }
+
     public DataType getDataType() {
         return _dataType;
     }