Skip to content

Commit

Permalink
Merge branch 'HDFGroup:develop' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
hyoklee authored Apr 18, 2024
2 parents 750b232 + bbf1e26 commit 73f1074
Show file tree
Hide file tree
Showing 52 changed files with 2,623 additions and 662 deletions.
5 changes: 3 additions & 2 deletions .github/workflows/release-files.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,16 @@ jobs:
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 0
ref: '${{ github.head_ref || github.ref_name }}'

- uses: rickstaa/action-create-tag@a1c7777fcb2fee4f19b0f283ba888afa11678b72 # v1.7.2
id: "tag_create"
with:
commit_sha: ${{ inputs.file_sha }}
tag: "${{ inputs.use_tag }}"
force_push_tag: true
force_push_tag: false
tag_exists_error: false
message: "Latest snapshot"
if: ${{ inputs.use_environ == 'snapshots' }}

# Print result using the action output.
- run: |
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/tarball.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ jobs:
uses: actions/[email protected]
with:
path: hdfsrc
ref: '${{needs.check_commits.outputs.branch_ref }}'

- name: Install Autotools Dependencies (Linux, serial)
run: |
Expand Down
72 changes: 36 additions & 36 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -533,42 +533,6 @@ if (BUILD_STATIC_EXECS)
endif ()
endif ()

option (HDF5_ENABLE_ANALYZER_TOOLS "enable the use of Clang tools" OFF)
mark_as_advanced (HDF5_ENABLE_ANALYZER_TOOLS)
if (HDF5_ENABLE_ANALYZER_TOOLS)
include (${HDF5_SOURCE_DIR}/config/sanitizer/tools.cmake)
endif ()
option (HDF5_ENABLE_SANITIZERS "execute the Clang sanitizer" OFF)
mark_as_advanced (HDF5_ENABLE_SANITIZERS)
if (HDF5_ENABLE_SANITIZERS)
include (${HDF5_SOURCE_DIR}/config/sanitizer/sanitizers.cmake)
endif ()
option (HDF5_ENABLE_FORMATTERS "format source files" OFF)
mark_as_advanced (HDF5_ENABLE_FORMATTERS)
if (HDF5_ENABLE_FORMATTERS)
include (${HDF5_SOURCE_DIR}/config/sanitizer/formatting.cmake)
endif ()

#-----------------------------------------------------------------------------
# Option to use code coverage
#-----------------------------------------------------------------------------
option (HDF5_ENABLE_COVERAGE "Enable code coverage for Libraries and Programs" OFF)
if (HDF5_ENABLE_COVERAGE)
include (${HDF5_SOURCE_DIR}/config/sanitizer/code-coverage.cmake)
if(CODE_COVERAGE AND CODE_COVERAGE_ADDED)
add_code_coverage () # Adds instrumentation to all targets
else ()
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 --coverage -fprofile-arcs -ftest-coverage")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g --coverage -O0 -fprofile-arcs -ftest-coverage")
if (CMAKE_C_COMPILER_ID STREQUAL "GNU")
set (LDFLAGS "${LDFLAGS} -fprofile-arcs -ftest-coverage")
link_libraries (gcov)
else ()
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
endif ()
endif ()
endif ()

#-----------------------------------------------------------------------------
# Option to indicate using a memory checker
#-----------------------------------------------------------------------------
Expand Down Expand Up @@ -1183,4 +1147,40 @@ if (EXISTS "${HDF5_SOURCE_DIR}/HDF5Examples" AND IS_DIRECTORY "${HDF5_SOURCE_DIR
endif ()
endif ()

option (HDF5_ENABLE_ANALYZER_TOOLS "enable the use of Clang tools" OFF)
mark_as_advanced (HDF5_ENABLE_ANALYZER_TOOLS)
if (HDF5_ENABLE_ANALYZER_TOOLS)
include (${HDF5_SOURCE_DIR}/config/sanitizer/tools.cmake)
endif ()
option (HDF5_ENABLE_SANITIZERS "execute the Clang sanitizer" OFF)
mark_as_advanced (HDF5_ENABLE_SANITIZERS)
if (HDF5_ENABLE_SANITIZERS)
include (${HDF5_SOURCE_DIR}/config/sanitizer/sanitizers.cmake)
endif ()
option (HDF5_ENABLE_FORMATTERS "format source files" OFF)
mark_as_advanced (HDF5_ENABLE_FORMATTERS)
if (HDF5_ENABLE_FORMATTERS)
include (${HDF5_SOURCE_DIR}/config/sanitizer/formatting.cmake)
endif ()

#-----------------------------------------------------------------------------
# Option to use code coverage
#-----------------------------------------------------------------------------
option (HDF5_ENABLE_COVERAGE "Enable code coverage for Libraries and Programs" OFF)
if (HDF5_ENABLE_COVERAGE)
include (${HDF5_SOURCE_DIR}/config/sanitizer/code-coverage.cmake)
if(CODE_COVERAGE AND CODE_COVERAGE_ADDED)
add_code_coverage () # Adds instrumentation to all targets
else ()
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 --coverage -fprofile-arcs -ftest-coverage")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g --coverage -O0 -fprofile-arcs -ftest-coverage")
if (CMAKE_C_COMPILER_ID STREQUAL "GNU")
set (LDFLAGS "${LDFLAGS} -fprofile-arcs -ftest-coverage")
link_libraries (gcov)
else ()
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
endif ()
endif ()
endif ()

include (CMakeInstallation.cmake)
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ Please make sure that you check the items applicable to your pull request:
* [ ] If changes were done to Autotools build, were they added to CMake and vice versa?
* [ ] Is the pull request applicable to any other branches? If yes, which ones? Please document it in the GitHub issue.
* [ ] Is the new code sufficiently documented for future maintenance?
* [ ] Does the new feature require a change to an existing API? See "API Compatibility Macros" document (https://docs.hdfgroup.org/hdf5/develop/api-compat-macros.html)
* [ ] Does the new feature require a change to an existing API? See "API Compatibility Macros" document (https://hdfgroup.github.io/hdf5/develop/api-compat-macros.html)
* Documentation
* [ ] Was the change described in the release_docs/RELEASE.txt file?
* [ ] Was the new function documented in the corresponding public header file using [Doxygen](https://hdfgroup.github.io/hdf5/develop/_r_m_t.html)?
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@ DOCUMENTATION
-------------
This release is fully functional for the API described in the documentation.

https://docs.hdfgroup.org/hdf5/develop/_l_b_a_p_i.html
https://hdfgroup.github.io/hdf5/develop/_l_b_a_p_i.html

Full Documentation and Programming Resources for this release can be found at

https://docs.hdfgroup.org/hdf5/develop/index.html
https://hdfgroup.github.io/hdf5/develop/index.html

The latest doxygen documentation generated on changes to develop is available at:

https://hdfgroup.github.io/hdf5/
https://hdfgroup.github.io/hdf5/develop

See the [RELEASE.txt](release_docs/RELEASE.txt) file in the [release_docs/](release_docs/) directory for information specific
to the features and updates included in this release of the library.
Expand Down
28 changes: 14 additions & 14 deletions doc/parallel-compression.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ participate in the collective write call.
## Multi-dataset I/O support

The parallel compression feature is supported when using the
multi-dataset I/O API routines ([H5Dwrite_multi](https://hdfgroup.github.io/hdf5/group___h5_d.html#gaf6213bf3a876c1741810037ff2bb85d8)/[H5Dread_multi](https://hdfgroup.github.io/hdf5/group___h5_d.html#ga8eb1c838aff79a17de385d0707709915)), but the
multi-dataset I/O API routines ([H5Dwrite_multi](https://hdfgroup.github.io/hdf5/develop/group___h5_d.html#gaf6213bf3a876c1741810037ff2bb85d8)/[H5Dread_multi](https://hdfgroup.github.io/hdf5/develop/group___h5_d.html#ga8eb1c838aff79a17de385d0707709915)), but the
following should be kept in mind:

- Parallel writes to filtered datasets **must** still be collective,
Expand All @@ -99,7 +99,7 @@ following should be kept in mind:

## Incremental file space allocation support

HDF5's [file space allocation time](https://hdfgroup.github.io/hdf5/group___d_c_p_l.html#ga85faefca58387bba409b65c470d7d851)
HDF5's [file space allocation time](https://hdfgroup.github.io/hdf5/develop/group___d_c_p_l.html#ga85faefca58387bba409b65c470d7d851)
is a dataset creation property that can have significant effects
on application performance, especially if the application uses
parallel HDF5. In a serial HDF5 application, the default file space
Expand All @@ -118,15 +118,15 @@ While this strategy has worked in the past, it has some noticeable
drawbacks. For one, the larger the chunked dataset being created,
the more noticeable overhead there will be during dataset creation
as all of the data chunks are being allocated in the HDF5 file.
Further, these data chunks will, by default, be [filled](https://hdfgroup.github.io/hdf5/group___d_c_p_l.html#ga4335bb45b35386daa837b4ff1b9cd4a4)
Further, these data chunks will, by default, be [filled](https://hdfgroup.github.io/hdf5/develop/group___d_c_p_l.html#ga4335bb45b35386daa837b4ff1b9cd4a4)
with HDF5's default fill data value, leading to extraordinary
dataset creation overhead and resulting in pre-filling large
portions of a dataset that the application might have been planning
to overwrite anyway. Even worse, there will be more initial overhead
from compressing that fill data before writing it out, only to have
it read back in, unfiltered and modified the first time a chunk is
written to. In the past, it was typically suggested that parallel
HDF5 applications should use [H5Pset_fill_time](https://hdfgroup.github.io/hdf5/group___d_c_p_l.html#ga6bd822266b31f86551a9a1d79601b6a2)
HDF5 applications should use [H5Pset_fill_time](https://hdfgroup.github.io/hdf5/develop/group___d_c_p_l.html#ga6bd822266b31f86551a9a1d79601b6a2)
with a value of `H5D_FILL_TIME_NEVER` in order to disable writing of
the fill value to dataset chunks, but this isn't ideal if the
application actually wishes to make use of fill values.
Expand Down Expand Up @@ -220,14 +220,14 @@ chunks to end up at addresses in the file that do not align
well with the underlying file system, possibly leading to
poor performance. As an example, Lustre performance is generally
good when writes are aligned with the chosen stripe size.
The HDF5 application can use [H5Pset_alignment](https://hdfgroup.github.io/hdf5/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a)
The HDF5 application can use [H5Pset_alignment](https://hdfgroup.github.io/hdf5/develop/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a)
to have a bit more control over where objects in the HDF5
file end up. However, do note that setting the alignment
of objects generally wastes space in the file and has the
potential to dramatically increase its resulting size, so
caution should be used when choosing the alignment parameters.

[H5Pset_alignment](https://hdfgroup.github.io/hdf5/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a)
[H5Pset_alignment](https://hdfgroup.github.io/hdf5/develop/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a)
has two parameters that control the alignment of objects in
the HDF5 file, the "threshold" value and the alignment
value. The threshold value specifies that any object greater
Expand Down Expand Up @@ -264,19 +264,19 @@ in a file, this can create significant amounts of free space
in the file over its lifetime and eventually cause performance
issues.

An HDF5 application can use [H5Pset_file_space_strategy](https://hdfgroup.github.io/hdf5/group___f_c_p_l.html#ga167ff65f392ca3b7f1933b1cee1b9f70)
An HDF5 application can use [H5Pset_file_space_strategy](https://hdfgroup.github.io/hdf5/develop/group___f_c_p_l.html#ga167ff65f392ca3b7f1933b1cee1b9f70)
with a value of `H5F_FSPACE_STRATEGY_PAGE` to enable the paged
aggregation feature, which can accumulate metadata and raw
data for dataset data chunks into well-aligned, configurably
sized "pages" for better performance. However, note that using
the paged aggregation feature will cause any setting from
[H5Pset_alignment](https://hdfgroup.github.io/hdf5/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a)
[H5Pset_alignment](https://hdfgroup.github.io/hdf5/develop/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a)
to be ignored. While an application should be able to get
comparable performance effects by [setting the size of these pages](https://hdfgroup.github.io/hdf5/group___f_c_p_l.html#gad012d7f3c2f1e1999eb1770aae3a4963) to be equal to the value that
would have been set for [H5Pset_alignment](https://hdfgroup.github.io/hdf5/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a),
comparable performance effects by [setting the size of these pages](https://hdfgroup.github.io/hdf5/develop/group___f_c_p_l.html#gad012d7f3c2f1e1999eb1770aae3a4963) to be equal to the value that
would have been set for [H5Pset_alignment](https://hdfgroup.github.io/hdf5/develop/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a),
this may not necessarily be the case and should be studied.

Note that [H5Pset_file_space_strategy](https://hdfgroup.github.io/hdf5/group___f_c_p_l.html#ga167ff65f392ca3b7f1933b1cee1b9f70)
Note that [H5Pset_file_space_strategy](https://hdfgroup.github.io/hdf5/develop/group___f_c_p_l.html#ga167ff65f392ca3b7f1933b1cee1b9f70)
has a `persist` parameter. This determines whether or not the
file free space manager should include extra metadata in the
HDF5 file about free space sections in the file. If this
Expand All @@ -300,12 +300,12 @@ hid_t file_id = H5Fcreate("file.h5", H5F_ACC_TRUNC, fcpl_id, fapl_id);

While the parallel compression feature requires that the HDF5
application set and maintain collective I/O at the application
interface level (via [H5Pset_dxpl_mpio](https://hdfgroup.github.io/hdf5/group___d_x_p_l.html#ga001a22b64f60b815abf5de8b4776f09e)),
interface level (via [H5Pset_dxpl_mpio](https://hdfgroup.github.io/hdf5/develop/group___d_x_p_l.html#ga001a22b64f60b815abf5de8b4776f09e)),
it does not require that the actual MPI I/O that occurs at
the lowest layers of HDF5 be collective; independent I/O may
perform better depending on the application I/O patterns and
parallel file system performance, among other factors. The
application may use [H5Pset_dxpl_mpio_collective_opt](https://hdfgroup.github.io/hdf5/group___d_x_p_l.html#gacb30d14d1791ec7ff9ee73aa148a51a3)
application may use [H5Pset_dxpl_mpio_collective_opt](https://hdfgroup.github.io/hdf5/develop/group___d_x_p_l.html#gacb30d14d1791ec7ff9ee73aa148a51a3)
to control this setting and see which I/O method provides the
best performance.

Expand All @@ -318,7 +318,7 @@ H5Dwrite(..., dxpl_id, ...);

### Runtime HDF5 Library version

An HDF5 application can use the [H5Pset_libver_bounds](https://hdfgroup.github.io/hdf5/group___f_a_p_l.html#gacbe1724e7f70cd17ed687417a1d2a910)
An HDF5 application can use the [H5Pset_libver_bounds](https://hdfgroup.github.io/hdf5/develop/group___f_a_p_l.html#gacbe1724e7f70cd17ed687417a1d2a910)
routine to set the upper and lower bounds on library versions
to use when creating HDF5 objects. For parallel compression
specifically, setting the library version to the latest available
Expand Down
Loading

0 comments on commit 73f1074

Please sign in to comment.