-
Notifications
You must be signed in to change notification settings - Fork 54
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
v1.78.0 -- Improve saving performance of SNIRF files by using low lev…
…el HDF5 library. (#172) v1.78.0 * DataTree, v1.13.0 -- Comprehensive solution to improving the save performance of SNIRF files by using low-level HDF5 calls and limiting the number of HDF5 open and create calls that are made for file, group and dataset.
- Loading branch information
Showing
12 changed files
with
316 additions
and
216 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
248 changes: 142 additions & 106 deletions
248
DataTree/AcquiredData/DataFiles/Hdf5/hdf5write_safe.m
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,122 +1,158 @@ | ||
| function err = hdf5write_safe(fname, name, val, options) | ||
|
|
||
| function err = hdf5write_safe(fileobj, name, val, options) | ||
| err = -1; | ||
| if isempty(val) | ||
| return; | ||
| end | ||
| if ~exist('options','var') | ||
| options = ''; | ||
| end | ||
|
|
||
| force_scalar = false; | ||
| force_array = false; | ||
| if any(strcmp(options, 'array')) | ||
| force_array = true; | ||
| elseif any(strcmp(options, 'scalar')) | ||
| force_scalar = true; | ||
| end | ||
|
|
||
| % Identify type of val and use SNIRF v1.1-compliant write function | ||
| fid = HDF5_GetFileDescriptor(fileobj); | ||
| if fid < 0 | ||
| err = -1; | ||
| if isempty(val) | ||
| return; | ||
| end | ||
| if ~exist('options','var') | ||
| options = ''; | ||
| end | ||
|
|
||
| force_scalar = false; | ||
| force_array = false; | ||
| if any(strcmp(options, 'array')) | ||
| force_array = true; | ||
| elseif any(strcmp(options, 'scalar')) | ||
| force_scalar = true; | ||
| end | ||
| % Identify type of val and use SNIRF v1.1-compliant write function | ||
|
|
||
| if exist(fname,'file') | ||
| fid = H5F.open(fname, 'H5F_ACC_RDWR', 'H5P_DEFAULT'); | ||
| return; | ||
| end | ||
|
|
||
| % Create group where dataset is located | ||
| group = filesepStandard(fileparts(name), 'nameonly'); | ||
| gid = HDF5_CreateGroup(fid, group); | ||
| if gid < 0 | ||
| err = -1; | ||
| return; | ||
| end | ||
|
|
||
| % Create dataset | ||
| if iscell(val) || isstring(val) | ||
| if length(val) > 1 && ~force_scalar || force_array % Returns true for single strings, believe it or not | ||
| write_string_array(fid, name, val); | ||
| else | ||
| fid = H5F.create(fname, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT'); | ||
| end | ||
| if fid < 0 | ||
| err = -1; | ||
| return; | ||
| write_string(fid, name, val); | ||
| end | ||
|
|
||
| % Create group where dataset is located | ||
| group = filesepStandard(fileparts(name), 'nameonly'); | ||
| gid = HDF5_CreateGroup(fid, group); | ||
| if gid < 0 | ||
| err = -1; | ||
| return; | ||
| elseif ischar(val) | ||
| write_string(fid, name, val); | ||
| elseif isfloat(val) | ||
| if length(val) > 1 && ~force_scalar || force_array | ||
| write_numeric_array(fid, name, val); | ||
| else | ||
| write_numeric(fid, name, val); | ||
| end | ||
|
|
||
| if iscell(val) || isstring(val) | ||
| if length(val) > 1 && ~force_scalar || force_array % Returns true for single strings, believe it or not | ||
| write_string_array(fid, fname, name, val); | ||
| else | ||
| write_string(fid, fname, name, val); | ||
| end | ||
| return | ||
| elseif ischar(val) | ||
| write_string(fid, fname, name, val); | ||
| return | ||
| elseif isfloat(val) | ||
| if length(val) > 1 && ~force_scalar || force_array | ||
| write_numeric_array(fname, name, val); | ||
| else | ||
| write_numeric(fid, fname, name, val); | ||
| end | ||
| return | ||
| elseif isinteger(val) | ||
| if length(val) > 1 && ~force_scalar || force_array | ||
| write_numeric_array(fid, fname, name, val); % As of now, no integer arrays exist | ||
| else | ||
| write_integer(fid, fname, name, val); | ||
| end | ||
| return | ||
| elseif isinteger(val) | ||
| if length(val) > 1 && ~force_scalar || force_array | ||
| write_numeric_array(fid, name, val); % As of now, no integer arrays exist | ||
| else | ||
| warning(['An unrecognized variable was saved to ', name, ' in ', fname]) | ||
| write_integer(fid, name, val); | ||
| end | ||
|
|
||
| H5F.close(fid); | ||
|
|
||
| else | ||
| warning(['An unrecognized variable was saved to ', name]) | ||
| end | ||
|
|
||
| function err = write_string(fid, fname, name, val) | ||
| sid = H5S.create('H5S_SCALAR'); | ||
| tid = H5T.copy('H5T_C_S1'); | ||
| H5T.set_size(tid, 'H5T_VARIABLE'); | ||
| did = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT'); | ||
| H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', {val}); | ||
| err = 0; | ||
| end | ||
|
|
||
| function err = write_string_array(fid, fname, name, val) | ||
| val = HDF5_Transpose(val); | ||
| sid = H5S.create_simple(1, numel(val), H5ML.get_constant_value('H5S_UNLIMITED')); | ||
| tid = H5T.copy('H5T_C_S1'); | ||
| H5T.set_size(tid, 'H5T_VARIABLE'); | ||
| pid = H5P.create('H5P_DATASET_CREATE'); | ||
| H5P.set_chunk(pid, 2); | ||
| did = H5D.create(fid, name, tid, sid, pid); | ||
| H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val); | ||
| err = 0; | ||
| end | ||
|
|
||
| function err = write_numeric(fid, fname, name, val) | ||
| fid = H5F.open(fname, 'H5F_ACC_RDWR', 'H5P_DEFAULT'); | ||
| tid = H5T.copy('H5T_NATIVE_DOUBLE'); | ||
| sid = H5S.create('H5S_SCALAR'); | ||
| H5D.create(fid, name, tid, sid, 'H5P_DEFAULT'); | ||
| h5write(fname, name, val); | ||
| err = 0; | ||
| % ----------------------------------------------------------------- | ||
| function err = write_string(fid, name, val) | ||
| sid = H5S.create('H5S_SCALAR'); | ||
| tid = H5T.copy('H5T_C_S1'); | ||
| H5T.set_size(tid, 'H5T_VARIABLE'); | ||
| did = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT'); | ||
| H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', {val}); | ||
| err = 0; | ||
|
|
||
|
|
||
|
|
||
| % ----------------------------------------------------------------- | ||
| function err = write_string_array(fid, name, val) | ||
| val = HDF5_Transpose(val); | ||
| sid = H5S.create_simple(1, numel(val), H5ML.get_constant_value('H5S_UNLIMITED')); | ||
| tid = H5T.copy('H5T_C_S1'); | ||
| H5T.set_size(tid, 'H5T_VARIABLE'); | ||
| pid = H5P.create('H5P_DATASET_CREATE'); | ||
| H5P.set_chunk(pid, 2); | ||
| dsid = H5D.create(fid, name, tid, sid, pid); | ||
| H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val); | ||
| err = 0; | ||
|
|
||
|
|
||
|
|
||
| % ----------------------------------------------------------------- | ||
| function err = write_numeric(fid, name, val) | ||
| tid = H5T.copy('H5T_NATIVE_DOUBLE'); | ||
| sid = H5S.create('H5S_SCALAR'); | ||
| dsid = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT'); | ||
| H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val); | ||
| err = 0; | ||
|
|
||
|
|
||
|
|
||
| % ----------------------------------------------------------------- | ||
| function err = write_integer(fid, name, val) | ||
| warning off; % Suppress the int truncation warning | ||
| tid = H5T.copy('H5T_NATIVE_ULONG'); | ||
| sid = H5S.create('H5S_SCALAR'); | ||
| dsid = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT'); | ||
| H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', int32(val)); | ||
| err = 0; | ||
| warning on; | ||
|
|
||
|
|
||
|
|
||
| % ----------------------------------------------------------------- | ||
| function err = write_numeric_array(fid, name, data) | ||
| err = 0; | ||
| data = HDF5_Transpose(data); | ||
| sizedata = size(data); | ||
| if sizedata(1) == 1 || sizedata(2) == 1 | ||
| n = length(data); | ||
| else | ||
| n = sizedata; | ||
| end | ||
|
|
||
| function err = write_numeric_array(fname, name, val) | ||
| val = HDF5_Transpose(val); | ||
| sizeval = size(val); | ||
| if sizeval(1) == 1 || sizeval(2) == 1 | ||
| n = length(val); | ||
| else | ||
| n = sizeval; | ||
| end | ||
| h5create(fname, name, n, 'Datatype', 'double'); | ||
| h5write(fname, name, val); | ||
| err = 0; | ||
| tid = -1; | ||
| sid = -1; | ||
| gid = -1; | ||
| dsid = -1; | ||
|
|
||
| maxdims = n; | ||
| try | ||
|
|
||
| sid = H5S.create_simple(numel(n), fliplr(n), fliplr(maxdims)); | ||
| gid = HDF5_CreateGroup(fid, fileparts(name)); | ||
| dsid = H5D.create(gid, name, 'H5T_NATIVE_DOUBLE', sid, 'H5P_DEFAULT'); | ||
| H5D.write(dsid, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', data); | ||
|
|
||
| catch | ||
|
|
||
| % Clean up; Close everything | ||
| cleanUp(tid, sid, gid, dsid); | ||
| err = -1; | ||
| return; | ||
|
|
||
| end | ||
| cleanUp(tid, sid, gid, dsid); | ||
|
|
||
|
|
||
| function err = write_integer(fid, fname, name, val) | ||
| warning off; % Suppress the int truncation warning | ||
| tid = H5T.copy('H5T_NATIVE_INT'); | ||
| sid = H5S.create('H5S_SCALAR'); | ||
| H5D.create(fid, name, tid, sid, 'H5P_DEFAULT'); | ||
| h5write(fname, name, val); | ||
| err = 0; | ||
| warning on; | ||
|
|
||
| % ------------------------------------------------------ | ||
| function cleanUp(tid, sid, gid, dsid) | ||
| if ~isnumeric(tid) | ||
| H5T.close(tid); | ||
| end | ||
| if ~isnumeric(sid) | ||
| H5S.close(sid); | ||
| end | ||
| if ~isnumeric(gid) | ||
| H5G.close(gid); | ||
| end | ||
| if ~isnumeric(dsid) | ||
| H5D.close(dsid); | ||
| end | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.