Feature supporting of different charsets

cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf4/H4header.java

@@ -10,6 +10,7 @@
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -104,7 +105,29 @@ public static void useHdfEos(boolean val) {
   private Map<Short, Vinfo> refnoMap = new HashMap<>();
 
   private MemTracker memTracker;
-  private PrintWriter debugOut = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8));
+  private PrintWriter debugOut;
+
+  private final Charset valueCharset;
+
+  public H4header() {
+    valueCharset = StandardCharsets.UTF_8;
+    debugOut = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8));
+  }
+
+  H4header(H4iosp h4iosp) {
+    valueCharset = h4iosp.getValueCharset().orElse(StandardCharsets.UTF_8);
+    debugOut = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8));
+  }
+
+  /**
+   * Return defined {@link Charset value charset} that
+   * will be used by reading HDF4 header.
+   * 
+   * @return {@link Charset value charset}
+   */
+  protected Charset getValueCharset() {
+    return valueCharset;
+  }
 
   public boolean isEos() {
     return isEos;
@@ -429,11 +452,11 @@ private Attribute makeAttribute(TagVH vh) throws IOException {
       case 3:
       case 4:
         if (nelems == 1)
-          att = new Attribute(name, raf.readStringMax(size));
+          att = new Attribute(name, raf.readStringMax(size, valueCharset));
         else {
           String[] vals = new String[nelems];
           for (int i = 0; i < nelems; i++)
-            vals[i] = raf.readStringMax(size);
+            vals[i] = raf.readStringMax(size, valueCharset);
           att = new Attribute(name, Array.factory(DataType.STRING, new int[] {nelems}, vals));
         }
         break;
@@ -1139,7 +1162,7 @@ List<DataChunk> readChunks(NetcdfFile ncfile) throws IOException {
 
     String read() throws IOException {
       raf.seek(data.offset);
-      return raf.readString(data.length);
+      return raf.readString(data.length, valueCharset);
     }
 
     public String toString() {
@@ -1661,7 +1684,7 @@ private class TagText extends Tag {
 
     protected void read() throws IOException {
       raf.seek(offset);
-      text = raf.readStringMax(length);
+      text = raf.readStringMax(length, valueCharset);
     }
 
     public String detail() {
@@ -1683,7 +1706,7 @@ protected void read() throws IOException {
       raf.seek(offset);
       obj_tagno = raf.readShort();
       obj_refno = raf.readShort();
-      text = raf.readStringMax(length - 4).trim();
+      text = raf.readStringMax(length - 4, valueCharset).trim();
     }
 
     public String detail() {
@@ -1840,7 +1863,7 @@ protected void read(int n) throws IOException {
       int start = 0;
       for (int i = 0; i < length; i++) {
         if (b[i] == 0) {
-          text[count] = new String(b, start, i - start, StandardCharsets.UTF_8);
+          text[count] = new String(b, start, i - start, valueCharset);
           count++;
           if (count == n)
             break;

cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf4/H4iosp.java

@@ -9,7 +9,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.util.List;
+import java.util.Optional;
 import ucar.ma2.Array;
 import ucar.ma2.ArrayStructure;
 import ucar.ma2.ArrayStructureBB;
@@ -35,13 +37,15 @@
 import ucar.nc2.util.IO;
 import ucar.unidata.io.PositioningDataInputStream;
 import ucar.unidata.io.RandomAccessFile;
+import javax.annotation.Nullable;
 
 /** HDF4 iosp */
 public class H4iosp extends AbstractIOServiceProvider {
   private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(H4iosp.class);
   private static boolean showLayoutTypes;
 
-  private H4header header = new H4header();
+  private H4header header;
+  private Charset valueCharset;
 
   @Override
   public boolean isValidFile(RandomAccessFile raf) throws IOException {
@@ -50,8 +54,9 @@ public boolean isValidFile(RandomAccessFile raf) throws IOException {
 
   @Override
   public String getFileTypeId() {
-    if (header.isEos())
+    if (header != null && header.isEos()) {
       return "HDF4-EOS";
+    }
     return DataFormatType.HDF4.getDescription();
   }
 
@@ -64,11 +69,24 @@ public String getFileTypeDescription() {
   public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
     super.open(raf, ncfile, cancelTask);
     Group.Builder rootGroup = Group.builder(null).setName("").setNcfile(ncfile);
-    header.read(raf, rootGroup, null);
+    getHeader().read(raf, rootGroup, null);
     ncfile.setRootGroup(rootGroup.build(null));
     ncfile.finish();
   }
 
+  /**
+   * Return header for reading netcdf file.
+   * Create it if it's not already created.
+   * 
+   * @return header for reading HDF4 file.
+   */
+  private H4header getHeader() {
+    if (header == null) {
+      header = new H4header(this);
+    }
+    return header;
+  }
+
   @Override
   public boolean isBuilder() {
     return true;
@@ -79,7 +97,7 @@ public void build(RandomAccessFile raf, Group.Builder rootGroup, CancelTask canc
     super.open(raf, rootGroup.getNcfile(), cancelTask);
 
     raf.order(RandomAccessFile.BIG_ENDIAN);
-    header = new H4header();
+    header = new H4header(this);
     header.read(raf, rootGroup, null);
   }
 
@@ -475,14 +493,36 @@ public ByteBuffer getByteBuffer() throws IOException {
   @Override
   public void reacquire() throws IOException {
     super.reacquire();
-    header.raf = this.raf;
+    getHeader().raf = this.raf;
   }
 
   public Object sendIospMessage(Object message) {
-    if (message.toString().equals("header"))
-      return header;
+    if (message instanceof Charset) {
+      setValueCharset((Charset) message);
+    }
+    if (message.toString().equals("header")) {
+      return getHeader();
+    }
     return super.sendIospMessage(message);
   }
 
+  /**
+   * Return {@link Charset value charset} if it was defined. Definition of charset
+   * occurs by sending a charset as a message using the {@link #sendIospMessage}
+   * method.
+   * 
+   * @return {@link Charset value charset} if it was defined.
+   */
+  protected Optional<Charset> getValueCharset() {
+    return Optional.ofNullable(valueCharset);
+  }
 
+  /**
+   * Define {@link Charset value charset}.
+   * 
+   * @param charset may be null.
+   */
+  protected void setValueCharset(@Nullable Charset charset) {
+    this.valueCharset = charset;
+  }
 }

cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java

@@ -15,6 +15,7 @@
 import java.nio.IntBuffer;
 import java.nio.LongBuffer;
 import java.nio.ShortBuffer;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Formatter;
@@ -177,10 +178,23 @@ public static boolean isValidFile(RandomAccessFile raf) throws IOException {
   private PrintWriter debugOut;
   private MemTracker memTracker;
 
+  private final Charset valueCharset;
+
   H5headerNew(RandomAccessFile myRaf, Group.Builder root, H5iospNew h5iosp) {
     this.raf = myRaf;
     this.root = root;
     this.h5iosp = h5iosp;
+    valueCharset = h5iosp.getValueCharset().orElse(StandardCharsets.UTF_8);
+  }
+
+  /**
+   * Return defined {@link Charset value charset} that
+   * will be used by reading HDF5 header.
+   *
+   * @return {@link Charset charset}
+   */
+  protected Charset getValueCharset() {
+    return valueCharset;
   }
 
   public void read(PrintWriter debugPS) throws IOException {
@@ -189,7 +203,7 @@ public void read(PrintWriter debugPS) throws IOException {
     } else if (debug1 || debugContinueMessage || debugCreationOrder || debugDetail || debugDimensionScales
         || debugGroupBtree || debugHardLink || debugHeap || debugPos || debugReference || debugTracker || debugV
         || debugSoftLink || warnings) {
-      debugOut = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8));
+      debugOut = new PrintWriter(new OutputStreamWriter(System.out));
     }
     h5objects = new H5objects(this, debugOut, memTracker);
 
@@ -1194,7 +1208,7 @@ private String convertString(byte[] b) {
         break;
       count++;
     }
-    return new String(b, 0, count, StandardCharsets.UTF_8); // all strings are considered to be UTF-8 unicode
+    return new String(b, 0, count, valueCharset); // all strings are considered to be UTF-8 unicode
   }
 
   private String convertString(byte[] b, int start, int len) {
@@ -1205,8 +1219,8 @@ private String convertString(byte[] b, int start, int len) {
         break;
       count++;
     }
-    return new String(b, start, count - start, StandardCharsets.UTF_8); // all strings are considered to be UTF-8
-                                                                        // unicode
+    return new String(b, start, count - start, valueCharset); // all strings are considered to be UTF-8
+    // unicode
   }
 
   protected Array convertEnums(Map<Integer, String> map, DataType dataType, Array values) {
@@ -2104,7 +2118,7 @@ String readHeapString(long heapIdAddress) throws IOException {
     if (ho.dataSize > 1000 * 1000)
       return String.format("Bad HeapObject.dataSize=%s", ho);
     raf.seek(ho.dataPos);
-    return raf.readString((int) ho.dataSize);
+    return raf.readString((int) ho.dataSize, valueCharset);
   }
 
   /**
@@ -2121,7 +2135,7 @@ String readHeapString(ByteBuffer bb, int pos) throws IOException {
     if (ho == null)
       throw new IllegalStateException("Cant find Heap Object,heapId=" + heapId);
     raf.seek(ho.dataPos);
-    return raf.readString((int) ho.dataSize);
+    return raf.readString((int) ho.dataSize, valueCharset);
   }
 
   Array readHeapVlen(ByteBuffer bb, int pos, DataType dataType, int endian) throws IOException, InvalidRangeException {

cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java

@@ -7,6 +7,8 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.charset.Charset;
+import java.util.Optional;
 import ucar.ma2.Array;
 import ucar.ma2.ArrayStructure;
 import ucar.ma2.ArrayStructureBB;
@@ -33,6 +35,7 @@
 import ucar.nc2.time.CalendarDate;
 import ucar.nc2.util.CancelTask;
 import ucar.unidata.io.RandomAccessFile;
+import javax.annotation.Nullable;
 
 /**
  * HDF5 I/O
@@ -105,6 +108,7 @@ public boolean isBuilder() {
   private H5headerNew header;
   private boolean isEos;
   boolean includeOriginalAttributes;
+  private Charset valueCharset;
 
   @Override
   public void build(RandomAccessFile raf, Group.Builder rootGroup, CancelTask cancelTask) throws IOException {
@@ -126,6 +130,34 @@ public void build(RandomAccessFile raf, Group.Builder rootGroup, CancelTask canc
     }
   }
 
+  @Override
+  public Object sendIospMessage(Object message) {
+    if (message instanceof Charset) {
+      setValueCharset((Charset) message);
+    }
+    return super.sendIospMessage(message);
+  }
+
+  /**
+   * Return {@link Charset value charset} if it was defined. Definition of charset
+   * occurs by sending a charset as a message using the {@link #sendIospMessage}
+   * method.
+   * 
+   * @return {@link Charset value charset} if it was defined.
+   */
+  protected Optional<Charset> getValueCharset() {
+    return Optional.ofNullable(valueCharset);
+  }
+
+  /**
+   * Define {@link Charset value charset}.
+   * 
+   * @param charset may be null.
+   */
+  protected void setValueCharset(@Nullable Charset charset) {
+    this.valueCharset = charset;
+  }
+
   @Override
   public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
     super.open(raf, ncfile, cancelTask);