From 1dfaef2586c7719cbbb228f9bcafdfa7100715ab Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Thu, 28 Mar 2024 11:20:57 -0500 Subject: [PATCH 01/12] replace deprecated CMAKE_COMPILER_IS_GNU** (#4272) --- HDF5Examples/config/cmake/HDFExampleMacros.cmake | 12 ++++++------ config/cmake/HDFCXXCompilerFlags.cmake | 6 +++--- config/cmake/HDFCompilerFlags.cmake | 4 ++-- config/cmake/LIBAEC/CMakeLists.txt | 2 +- config/cmake/ZLIB/CMakeLists.txt | 6 +++--- config/sanitizer/sanitizers.cmake | 15 +++++++++++++++ 6 files changed, 30 insertions(+), 15 deletions(-) diff --git a/HDF5Examples/config/cmake/HDFExampleMacros.cmake b/HDF5Examples/config/cmake/HDFExampleMacros.cmake index 5c425dbbe0c..8173562de88 100644 --- a/HDF5Examples/config/cmake/HDFExampleMacros.cmake +++ b/HDF5Examples/config/cmake/HDFExampleMacros.cmake @@ -45,10 +45,10 @@ macro (BASIC_SETTINGS varname) #----------------------------------------------------------------------------- # Compiler specific flags : Shouldn't there be compiler tests for these #----------------------------------------------------------------------------- - if (CMAKE_COMPILER_IS_GNUCC) + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}") endif () - if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_COMPILER_IS_GNUCXX) + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS}") endif () @@ -56,10 +56,10 @@ macro (BASIC_SETTINGS varname) # This is in here to help some of the GCC based IDES like Eclipse # and code blocks parse the compiler errors and warnings better. #----------------------------------------------------------------------------- - if (CMAKE_COMPILER_IS_GNUCC) + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fmessage-length=0") endif () - if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_COMPILER_IS_GNUCXX) + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmessage-length=0") endif () @@ -74,7 +74,7 @@ macro (BASIC_SETTINGS varname) set (HDF_WARNINGS_BLOCKED 1) string (REGEX REPLACE "(^| )([/-])W[0-9]( |$)" " " CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /w") - if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_COMPILER_IS_GNUCXX) + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") string (REGEX REPLACE "(^| )([/-])W[0-9]( |$)" " " CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /w") endif () @@ -91,7 +91,7 @@ macro (BASIC_SETTINGS varname) # Most compilers use -w to suppress warnings. if (NOT HDF_WARNINGS_BLOCKED) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") - if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_COMPILER_IS_GNUCXX) + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") endif () endif () diff --git a/config/cmake/HDFCXXCompilerFlags.cmake b/config/cmake/HDFCXXCompilerFlags.cmake index dd120c911cb..5f977f534f6 100644 --- a/config/cmake/HDFCXXCompilerFlags.cmake +++ b/config/cmake/HDFCXXCompilerFlags.cmake @@ -65,7 +65,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC" AND CMAKE_CXX_COMPILER_LOADED) endif () endif () -if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED) +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_LOADED) set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS}") if (${HDF_CFG_NAME} MATCHES "Debug" OR ${HDF_CFG_NAME} MATCHES "Developer") if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) @@ -143,7 +143,7 @@ else () endif() endif() elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) # add the general CXX flags for g++ compiler versions 4.8 and above. ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-general") @@ -311,7 +311,7 @@ endif () # This is in here to help some of the GCC based IDES like Eclipse # and code blocks parse the compiler errors and warnings better. #----------------------------------------------------------------------------- -if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED) +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_LOADED) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmessage-length=0") endif () diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake index 007747a6166..d8a444b84d2 100644 --- a/config/cmake/HDFCompilerFlags.cmake +++ b/config/cmake/HDFCompilerFlags.cmake @@ -96,7 +96,7 @@ if (CMAKE_C_COMPILER_ID STREQUAL "NVHPC" ) set (CMAKE_C_FLAGS_RELWITHDEBINFO "${cmake_c_flags_relwithdebinfo_edited}") endif () -if (CMAKE_COMPILER_IS_GNUCC) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}") if (${HDF_CFG_NAME} MATCHES "Debug" OR ${HDF_CFG_NAME} MATCHES "Developer") if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0) @@ -405,7 +405,7 @@ endif () # This is in here to help some of the GCC based IDES like Eclipse # and code blocks parse the compiler errors and warnings better. #----------------------------------------------------------------------------- -if (CMAKE_COMPILER_IS_GNUCC) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fmessage-length=0") endif () diff --git a/config/cmake/LIBAEC/CMakeLists.txt b/config/cmake/LIBAEC/CMakeLists.txt index 11f79414e2a..54482163c5e 100644 --- a/config/cmake/LIBAEC/CMakeLists.txt +++ b/config/cmake/LIBAEC/CMakeLists.txt @@ -134,7 +134,7 @@ endif () # This is in here to help some of the GCC based IDES like Eclipse # and code blocks parse the compiler errors and warnings better. #----------------------------------------------------------------------------- -if (CMAKE_COMPILER_IS_GNUCC) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fmessage-length=0") endif () diff --git a/config/cmake/ZLIB/CMakeLists.txt b/config/cmake/ZLIB/CMakeLists.txt index 529f1446a58..7b5d0cf62dd 100644 --- a/config/cmake/ZLIB/CMakeLists.txt +++ b/config/cmake/ZLIB/CMakeLists.txt @@ -128,7 +128,7 @@ endif () #----------------------------------------------------------------------------- # Compiler specific flags : Shouldn't there be compiler tests for these #----------------------------------------------------------------------------- -if (CMAKE_COMPILER_IS_GNUCC) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS} -Wno-strict-prototypes") endif () if (CMAKE_C_COMPILER_ID MATCHES "IntelLLVM" OR CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") @@ -139,7 +139,7 @@ endif () # This is in here to help some of the GCC based IDES like Eclipse # and code blocks parse the compiler errors and warnings better. #----------------------------------------------------------------------------- -if (CMAKE_COMPILER_IS_GNUCC) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fmessage-length=0") endif () @@ -196,7 +196,7 @@ if(NOT MINGW) ) endif() -if(CMAKE_COMPILER_IS_GNUCC) +if(CMAKE_C_COMPILER_ID STREQUAL "GNU") if(ASM686) set(ZLIB_ASMS contrib/asm686/match.S) elseif (AMD64) diff --git a/config/sanitizer/sanitizers.cmake b/config/sanitizer/sanitizers.cmake index 0afb2077910..72f101f4c25 100644 --- a/config/sanitizer/sanitizers.cmake +++ b/config/sanitizer/sanitizers.cmake @@ -57,9 +57,15 @@ if(USE_SANITIZER) if(UNIX) append("-fno-omit-frame-pointer" CMAKE_C_FLAGS) message(STATUS "Building with sanitize, base flags=${CMAKE_C_SANITIZER_FLAGS}") + if (CMAKE_CXX_COMPILER_LOADED) + append("-fno-omit-frame-pointer" CMAKE_CXX_FLAGS) + endif () if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") append("-O1" CMAKE_C_FLAGS) + if (CMAKE_CXX_COMPILER_LOADED) + append("-O1" CMAKE_CXX_FLAGS) + endif () endif() if(USE_SANITIZER MATCHES "([Aa]ddress)") @@ -176,6 +182,9 @@ if(USE_SANITIZER) if(SANITIZER_SELECTED_COMPATIBLE) message(STATUS " Building with ${SANITIZER_SELECTED_FLAGS}") append("${SANITIZER_SELECTED_FLAGS}" CMAKE_C_FLAGS) + if (CMAKE_CXX_COMPILER_LOADED) + append("${SANITIZER_SELECTED_FLAGS}" CMAKE_CXX_FLAGS) + endif () else() message(FATAL_ERROR "Unsupported value of USE_SANITIZER: ${USE_SANITIZER}") endif() @@ -183,6 +192,9 @@ if(USE_SANITIZER) if(USE_SANITIZER MATCHES "([Aa]ddress)") message(STATUS "Building with Address sanitizer") append("-fsanitize=address" CMAKE_C_FLAGS) + if (CMAKE_CXX_COMPILER_LOADED) + append("-fsanitize=address" CMAKE_CXX_FLAGS) + endif () if(AFL) append_quoteless(AFL_USE_ASAN=1 CMAKE_C_COMPILER_LAUNCHER) @@ -197,6 +209,9 @@ if(USE_SANITIZER) if(USE_SANITIZER MATCHES "([Aa]ddress)") message(STATUS "Building with Address sanitizer") append("/fsanitize=address" CMAKE_C_FLAGS) + if (CMAKE_CXX_COMPILER_LOADED) + append("/fsanitize=address" CMAKE_CXX_FLAGS) + endif () else() message(FATAL_ERROR "This sanitizer not yet supported in the MSVC environment: ${USE_SANITIZER}") endif() From 372381c530275a1616dce487cc578b19c4812163 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Thu, 28 Mar 2024 13:12:19 -0500 Subject: [PATCH 02/12] Prevent stack overflows in H5E__push_stack (#4264) --- src/H5Eint.c | 6 ++-- src/H5Iint.c | 74 ++++++++++++++++++++++++++++++++++++++++++++---- src/H5Iprivate.h | 1 + 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/H5Eint.c b/src/H5Eint.c index a4ba5b2d4b9..70848ecd7c2 100644 --- a/src/H5Eint.c +++ b/src/H5Eint.c @@ -730,13 +730,13 @@ H5E__push_stack(H5E_t *estack, const char *file, const char *func, unsigned line if (estack->nused < H5E_NSLOTS) { /* Increment the IDs to indicate that they are used in this stack */ - if (H5I_inc_ref(cls_id, false) < 0) + if (H5I_inc_ref_noherr(cls_id, false) < 0) HGOTO_DONE(FAIL); estack->slot[estack->nused].cls_id = cls_id; - if (H5I_inc_ref(maj_id, false) < 0) + if (H5I_inc_ref_noherr(maj_id, false) < 0) HGOTO_DONE(FAIL); estack->slot[estack->nused].maj_num = maj_id; - if (H5I_inc_ref(min_id, false) < 0) + if (H5I_inc_ref_noherr(min_id, false) < 0) HGOTO_DONE(FAIL); estack->slot[estack->nused].min_num = min_id; /* The 'func' & 'file' strings are statically allocated (by the compiler) diff --git a/src/H5Iint.c b/src/H5Iint.c index fe3b90c2454..1df3ae907a8 100644 --- a/src/H5Iint.c +++ b/src/H5Iint.c @@ -1230,6 +1230,27 @@ H5I_dec_app_ref_always_close_async(hid_t id, void **token) FUNC_LEAVE_NOAPI(ret_value) } /* end H5I_dec_app_ref_always_close_async() */ +/*------------------------------------------------------------------------- + * Function: H5I_do_inc_ref + * + * Purpose: Helper function for H5I_inc_ref/H5I_inc_ref_noherr to + * actually increment the reference count for an object. + * + * Return: The new reference count (can't fail) + * + *------------------------------------------------------------------------- + */ +static inline int +H5I_do_inc_ref(H5I_id_info_t *info, bool app_ref) +{ + /* Adjust reference counts */ + ++(info->count); + if (app_ref) + ++(info->app_count); + + return (int)(app_ref ? info->app_count : info->count); +} + /*------------------------------------------------------------------------- * Function: H5I_inc_ref * @@ -1255,18 +1276,59 @@ H5I_inc_ref(hid_t id, bool app_ref) if (NULL == (info = H5I__find_id(id))) HGOTO_ERROR(H5E_ID, H5E_BADID, (-1), "can't locate ID"); - /* Adjust reference counts */ - ++(info->count); - if (app_ref) - ++(info->app_count); - /* Set return value */ - ret_value = (int)(app_ref ? info->app_count : info->count); + ret_value = H5I_do_inc_ref(info, app_ref); done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5I_inc_ref() */ +/*------------------------------------------------------------------------- + * Function: H5I_inc_ref_noherr + * + * Purpose: Increment the reference count for an object. Exactly like + * H5I_inc_ref, except that it makes use of HGOTO_DONE on + * failure instead of HGOTO_ERROR. This function is + * specifically meant to be used in the H5E package, where we + * have to avoid calling any function or macro that may call + * HGOTO_ERROR and similar. Otherwise, we can cause a stack + * overflow that looks like (for example): + * + * H5E_printf_stack() + * H5E__push_stack() + * H5I_inc_ref() + * H5I__find_id() (FAIL) + * HGOTO_ERROR() + * H5E_printf_stack() + * ... + * + * Return: Success: The new reference count + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +int +H5I_inc_ref_noherr(hid_t id, bool app_ref) +{ + H5I_id_info_t *info = NULL; /* Pointer to the ID info */ + int ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI_NOERR + + /* Sanity check */ + assert(id >= 0); + + /* General lookup of the ID */ + if (NULL == (info = H5I__find_id(id))) + HGOTO_DONE((-1)); + + /* Set return value */ + ret_value = H5I_do_inc_ref(info, app_ref); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5I_inc_ref_noherr() */ + /*------------------------------------------------------------------------- * Function: H5I_get_ref * diff --git a/src/H5Iprivate.h b/src/H5Iprivate.h index 75a5787b616..83fdacc686f 100644 --- a/src/H5Iprivate.h +++ b/src/H5Iprivate.h @@ -68,6 +68,7 @@ H5_DLL H5I_type_t H5I_get_type(hid_t id); H5_DLL herr_t H5I_iterate(H5I_type_t type, H5I_search_func_t func, void *udata, bool app_ref); H5_DLL int H5I_get_ref(hid_t id, bool app_ref); H5_DLL int H5I_inc_ref(hid_t id, bool app_ref); +H5_DLL int H5I_inc_ref_noherr(hid_t id, bool app_ref); H5_DLL int H5I_dec_ref(hid_t id); H5_DLL int H5I_dec_app_ref(hid_t id); H5_DLL int H5I_dec_app_ref_async(hid_t id, void **token); From 03b6575b1b12326d709a007c07b45f098189842e Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Thu, 28 Mar 2024 13:49:50 -0500 Subject: [PATCH 03/12] Minor fixes after merge of file format security fixes (#4263) * Update H5_IS_BUFFER_OVERFLOW to account for 'size' of 0 * Invert a few checks to avoid function call --- src/H5Dint.c | 4 ++-- src/H5Oattribute.c | 4 ++-- src/H5Tcommit.c | 4 ++-- src/H5private.h | 21 +++++++++++++++------ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/H5Dint.c b/src/H5Dint.c index 37c9fe490a8..3b9d000f523 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -947,8 +947,8 @@ H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTPIN, FAIL, "unable to pin dataset object header"); /* Check for creating dataset with unusual datatype */ - if (H5T_is_numeric_with_unusual_unused_bits(type) && - !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + if (!(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)) && + H5T_is_numeric_with_unusual_unused_bits(type)) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "creating dataset with unusual datatype, see documentation for " "H5Pset_relax_file_integrity_checks for details."); diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c index 64634aac405..4929be56717 100644 --- a/src/H5Oattribute.c +++ b/src/H5Oattribute.c @@ -212,8 +212,8 @@ H5O__attr_create(const H5O_loc_t *loc, H5A_t *attr) HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header"); /* Check for creating attribute with unusual datatype */ - if (H5T_is_numeric_with_unusual_unused_bits(attr->shared->dt) && - !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(loc->file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + if (!(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(loc->file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)) && + H5T_is_numeric_with_unusual_unused_bits(attr->shared->dt)) HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "creating attribute with unusual datatype, see documentation for " "H5Pset_relax_file_integrity_checks for details."); diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index 2e23764fa7a..00a88984375 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -490,8 +490,8 @@ H5T__commit(H5F_t *file, H5T_t *type, hid_t tcpl_id) HGOTO_ERROR(H5E_ATTR, H5E_CANTPIN, FAIL, "unable to pin object header"); /* Check for creating committed datatype with unusual datatype */ - if (H5T_is_numeric_with_unusual_unused_bits(type) && - !(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS))) + if (!(H5O_has_chksum(oh) || (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)) && + H5T_is_numeric_with_unusual_unused_bits(type)) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "creating committed datatype with unusual datatype, see documentation for " "H5Pset_relax_file_integrity_checks for details."); diff --git a/src/H5private.h b/src/H5private.h index a32421bf2ed..c795704ece6 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -352,12 +352,21 @@ * For the time being, these can be suppressed with * H5_GCC_CLANG_DIAG_OFF("type-limits")/H5_GCC_CLANG_DIAG_ON("type-limits") */ +/* clang-format off */ #define H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end) \ - (((ptr) > (buffer_end)) || /* Bad precondition */ \ - (((size_t)(size) <= PTRDIFF_MAX) && \ - ((ptrdiff_t)(size) < 0)) || /* Account for (likely unintentional) negative 'size' */ \ - ((size_t)(size) > \ - (size_t)((((const uint8_t *)buffer_end) - ((const uint8_t *)ptr)) + 1))) /* Typical overflow */ + ( \ + /* Trivial case */ \ + ((size) != 0) && \ + ( \ + /* Bad precondition */ \ + ((ptr) > (buffer_end)) || \ + /* Account for (likely unintentional) negative 'size' */ \ + (((size_t)(size) <= PTRDIFF_MAX) && ((ptrdiff_t)(size) < 0)) || \ + /* Typical overflow */ \ + ((size_t)(size) > (size_t)((((const uint8_t *)buffer_end) - ((const uint8_t *)ptr)) + 1)) \ + ) \ + ) +/* clang-format on */ /* Variant of H5_IS_BUFFER_OVERFLOW, used with functions such as H5Tdecode() * that don't take a size parameter, where we need to skip the bounds checks. @@ -366,7 +375,7 @@ * the entire library. */ #define H5_IS_KNOWN_BUFFER_OVERFLOW(skip, ptr, size, buffer_end) \ - (skip ? false : ((ptr) + (size)-1) > (buffer_end)) + (skip ? false : H5_IS_BUFFER_OVERFLOW(ptr, size, buffer_end)) /* * HDF Boolean type. From c3d1c7c0abe114defc453401da84084801d5201f Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:07:52 -0700 Subject: [PATCH 04/12] CHECK --> CHECK_PTR in tmisc.c (#4274) --- test/tmisc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tmisc.c b/test/tmisc.c index edfdcd406c0..a9d94a5ec97 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -3982,7 +3982,7 @@ test_misc21(void) /* Allocate space for the buffer */ buf = (char *)calloc(MISC21_SPACE_DIM0 * MISC21_SPACE_DIM1, 1); - CHECK(buf, NULL, "calloc"); + CHECK_PTR(buf, "calloc"); /* Create the file */ fid = H5Fcreate(MISC21_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -4068,7 +4068,7 @@ test_misc22(void) /* Allocate space for the buffer */ buf = (char *)calloc(MISC22_SPACE_DIM0 * MISC22_SPACE_DIM1, 8); - CHECK(buf, NULL, "calloc"); + CHECK_PTR(buf, "calloc"); /* Create a file access property list */ fapl = H5Pcreate(H5P_FILE_ACCESS); From e908accadf92ce658572d928369f616e25a8cc9b Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:08:05 -0700 Subject: [PATCH 05/12] Add release note for CVE-2017-17507 (#4275) --- release_docs/RELEASE.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index d45d9d1ee9b..28dbd2357be 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -693,6 +693,19 @@ Bug Fixes since HDF5-1.14.0 release Library ------- + - Fixed CVE-2017-17507 + + This CVE was previously declared fixed, but later testing with a static + build of HDF5 showed that it was not fixed. + + When parsing a malformed (fuzzed) compound type containing variable-length + string members, the library could produce a segmentation fault, crashing + the library. + + This was fixed after GitHub PR #4234 + + Fixes GitHub issue #3446 + - Fixed a cache assert with very large metadata objects If the library tries to load a metadata object that is above a From 6c5412204254992334dc7af6f52e2ac3bad83885 Mon Sep 17 00:00:00 2001 From: "H. Joe Lee" Date: Thu, 28 Mar 2024 14:40:28 -0500 Subject: [PATCH 06/12] Update Cygwin installation guide (#4265) --- release_docs/INSTALL_Cygwin.txt | 87 +++++++++++++++------------------ 1 file changed, 40 insertions(+), 47 deletions(-) diff --git a/release_docs/INSTALL_Cygwin.txt b/release_docs/INSTALL_Cygwin.txt index 44516468c07..8f18ef98f72 100644 --- a/release_docs/INSTALL_Cygwin.txt +++ b/release_docs/INSTALL_Cygwin.txt @@ -2,20 +2,21 @@ HDF5 Build and Install Instructions for Cygwin ************************************************************************ -This document is an instruction on how to build, test and install HDF5 library on -Cygwin. See detailed information in hdf5/INSTALL. +This document is an instruction on how to build, test and install HDF5 library +on Cygwin. See detailed information in hdf5/INSTALL. -NOTE: hdf5 can be built with CMake, see the INSTALL_CMake.txt file for more guidance. +NOTE: hdf5 can be built with CMake, see the INSTALL_CMake.txt file for more +guidance. Preconditions: -------------- -1. Installed Cygwin 1.7.25 or higher +1. Cygwin 3.5.1 or higher Installed To install the Cygwin net release, go to http://www.cygwin.com and - click on "setup-x86.exe" (32-bit installation) under the heading + click on "setup-x86_64.exe" under the heading "Current Cygwin DLL version". This will download a GUI - installer called setup-x86.exe which can be run to download a complete + installer called setup-x86_64.exe which can be run to download a complete Cygwin installation via the internet. Then follow the instructions on each screen to install Cygwin. @@ -34,10 +35,10 @@ Preconditions: The following compilers are supported by HDF5 and included in the Cygwin package system: - gcc (4.7.3 and 4.9.2), which includes: - gcc4-core : C compiler - gcc4-g++ : C++ compiler - gcc4-fortran : fortran compiler + gcc, which includes: + gcc-core : C compiler + gcc-g++ : C++ compiler + gcc-fortran : Fortran compiler 2.1.1 Using Compilers Not Supported @@ -59,18 +60,13 @@ Preconditions: 2.2 HDF5 External Library Dependencies - 2.2.1 Zlib + 2.2.1 zlib - zlib-1.2.5 or later is supported and tested on Cygwin. + zlib-1.2.8 or later is supported and tested. 2.2.2 Szip - The HDF5 library has a predefined compression filter that uses - the extended-Rice lossless compression algorithm for chunked - datasets. For more information on Szip compression, license terms, - and obtaining the Szip source code, see: - - https://portal.hdfgroup.org/display/HDF5/Szip+Compression+in+HDF+Products + libaec-1.1.2 or later is supported and tested. 2.3 Additional Utilities @@ -91,8 +87,8 @@ Build, Test and Install HDF5 on Cygwin -------------------------------------- 1. Get HDF5 source code package - Users can download HDF5 source code package from HDF website - (http://hdfgroup.org). + Users can download the HDF5 source code from the official GitHub repository + (https://github.com/HDFGroup/hdf5). 2. Unpacking the distribution @@ -102,32 +98,30 @@ Build, Test and Install HDF5 on Cygwin 2.1 Non-compressed tar archive (*.tar) - $ tar xf hdf5-1.15.x.tar + $ tar xf hdf5-1.15.x.tar 2.2 Gzip'd tar archive (*.tar.gz) - $ gunzip < hdf5-1.15.x.tar.gz | tar xf - + $ gunzip < hdf5-1.15.x.tar.gz | tar xf - 2.3 Bzip'd tar archive (*.tar.bz2) - $ bunzip2 < hdf5-1.15.x.tar.bz2 | tar xf - + $ bunzip2 < hdf5-1.15.x.tar.bz2 | tar xf - 2. Setup Environment In Cygwin, most compilers and setting are automatically detected during - the configure script. However, if you are building Fortran we recommend + the configure script. However, if you are building Fortran, we recommend that you explicitly set the "FC" variable in your environment to use the - gfortran compiler. For example, issue the command: + gfortran compiler. For example, issue the command: - $ export FC=gfortran + $ export FC=gfortran 4. Configuring - Notes: See detailed information in hdf5/release_docs/INSTALL, - part 5. Full installation instructions for source - distributions + Notes: See detailed information in hdf5/release_docs/INSTALL_Auto.txt. - The host configuration file for cygwin i686-pc-cygwin is located + The host configuration file for Cygwin is located in the `config' directory and are based on architecture name, vendor name, and operating system which are displayed near the beginning of the `configure' output. The host config file influences @@ -137,34 +131,34 @@ Build, Test and Install HDF5 on Cygwin To configure HDF5 C Library, using - $ ./configure + $ ./configure To configure HDF5 C/C++ Library, using - $ ./configure --enable-cxx + $ ./configure --enable-cxx To configure HDF5 C/Fortran Library, using - $ ./configure --enable-fortran + $ ./configure --enable-fortran To configure HDF5 C with Szip library, using - $ ./configure --with-szlib="path to szlib" + $ ./configure --with-szlib="path to szlib" - For example, if szip library was installed in the directory + For example, if Szip library was installed in the directory /cygdrive/c/szip, which is parent directory of "include" and "lib", then the following command will configure HDF5 C library - with szip enabled: + with Szip enabled: - $ ./configure --with-szlib=/cygdrive/c/szip + $ ./configure --with-szlib=/cygdrive/c/szip - To configure HDF5 C without Zlib, + To configure HDF5 C without zlib, To disable zlib, using $ ./configure --without-zlib - Two ways to configure HDF5 C with specified Zlib + Two ways to configure HDF5 C with specified zlib Using @@ -174,7 +168,7 @@ Build, Test and Install HDF5 on Cygwin /cygdrive/c/usr, which is the parent directory of directories "include" and "lib", - $ ./configure --with-zlib=/cygdrive/c/usr/include,/cygdrive/c/usr/lib + $ ./configure --with-zlib=/cygdrive/c/usr/include,/cygdrive/c/usr/lib Through the CPPFLAGS and LDFLAGS Variables @@ -188,7 +182,7 @@ Build, Test and Install HDF5 on Cygwin To specify the installation directories, using - $ ./configure --prefix="path for installation" + $ ./configure --prefix="path for installation" By default, HDF5 library, header files, examples, and support programs will be installed in /usr/local/lib, @@ -201,7 +195,7 @@ Build, Test and Install HDF5 on Cygwin All of the above switches can be combined together. For example, if users want to configure HDF5 C/C++/Fortran - library with szip library enabled, with zlib library at + library with Szip library enabled, with zlib library at /cygdrive/c/usr/, and install HDF5 into directory /cygdrive/c/hdf5 using gcc/g++ as C/C++ compiler and gfortran as fortran compiler @@ -237,15 +231,15 @@ Build, Test and Install HDF5 on Cygwin After configuration is done successfully, run the following series of commands to build, test and install HDF5 - $ make > "output file name" - $ make check > "output file name" + $ make > "output file name" + $ make check > "output file name" Before run "make install", check output file for "make check", there should be no failures at all. 6. Make Install - $ make install > "output file name" + $ make install > "output file name" 7. Check installed HDF5 library @@ -255,8 +249,7 @@ Build, Test and Install HDF5 on Cygwin 8. Known Problems - dt_arith tests may fail due to the use of fork. This is a known issue - with cygwin on Windows. + cache_api tests may fail. This is a known issue with Cygwin. "make check" fails when building shared lib files is enabled. The default on Cygwin has been changed to disable shared. It can be enabled with From f6ae24103290215a92b981a205215e59a6ada5f3 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 28 Mar 2024 16:55:32 -0500 Subject: [PATCH 07/12] Addresses configuration fortran testing flags (#4276) * turn warnings to errors in fortran configure test * Intel fortran test fix --- config/cmake/HDF5UseFortran.cmake | 6 ++++++ fortran/testpar/subfiling.F90 | 2 +- m4/aclocal_fc.m4 | 7 ++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config/cmake/HDF5UseFortran.cmake b/config/cmake/HDF5UseFortran.cmake index 318f9a69848..b4172eace4b 100644 --- a/config/cmake/HDF5UseFortran.cmake +++ b/config/cmake/HDF5UseFortran.cmake @@ -218,6 +218,11 @@ if (${HAVE_ISO_FORTRAN_ENV}) string (REGEX REPLACE "," ";" VAR "${pac_validLogicalKinds}") set(CMAKE_REQUIRED_QUIET TRUE) + set(save_CMAKE_Fortran_FLAGS ${CMAKE_Fortran_FLAGS}) + if (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") + set(CMAKE_Fortran_FLAGS "-warn error") + endif () + foreach (KIND ${VAR}) unset(MPI_LOGICAL_KIND CACHE) set (PROG_SRC @@ -243,6 +248,7 @@ if (${HAVE_ISO_FORTRAN_ENV}) message (FATAL_ERROR "Failed to determine LOGICAL KIND for MPI") endif () set(CMAKE_REQUIRED_QUIET FALSE) + set(CMAKE_Fortran_FLAGS ${save_CMAKE_Fortran_FLAGS}) endif() endif() diff --git a/fortran/testpar/subfiling.F90 b/fortran/testpar/subfiling.F90 index a9c67e9b94c..67f201e0ec7 100644 --- a/fortran/testpar/subfiling.F90 +++ b/fortran/testpar/subfiling.F90 @@ -134,7 +134,7 @@ PROGRAM subfiling_test ENDIF CALL mpi_info_get(info_ret,"foo", 3_MPI_INTEGER_KIND, info_val, flag, mpierror) - IF(LOGICAL(flag) .EQV. .TRUE.)THEN + IF(LOGICAL(flag) .EQV. LOGICAL(.TRUE.))THEN IF(info_val.NE."bar")THEN IF(mpi_rank.EQ.0) & WRITE(*,*) "Failed H5Pset_mpi_params_f and H5Pget_mpi_params_f sequence" diff --git a/m4/aclocal_fc.m4 b/m4/aclocal_fc.m4 index e6a1f26ec35..cfcfbcf7ca2 100644 --- a/m4/aclocal_fc.m4 +++ b/m4/aclocal_fc.m4 @@ -391,7 +391,12 @@ AC_REQUIRE([PAC_FC_AVAIL_KINDS]) AC_MSG_CHECKING([default Fortran KIND of LOGICAL in MPI]) AC_LANG_PUSH([Fortran]) saved_FCFLAGS=$FCFLAGS -FCFLAGS="" +check_Intel="`$FC -V 2>&1 |grep '^Intel'`" +if test X != "X$check_Intel"; then + FCFLAGS="-warn error" +else + FCFLAGS="" +fi for kind in `echo $pac_validLogicalKinds | sed -e 's/,/ /g'`; do AC_COMPILE_IFELSE([ PROGRAM main From 16ba713e31363abd1692e6cf8f5bc4114a34e5bc Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:29:22 -0500 Subject: [PATCH 08/12] Merge julia workflows into standard ci format (#4273) --- .github/workflows/autotools.yml | 6 ++ .github/workflows/cmake.yml | 6 ++ .github/workflows/julia-auto.yml | 79 +++++++++++++++++++++++++++ .github/workflows/julia-cmake.yml | 82 ++++++++++++++++++++++++++++ .github/workflows/lin-auto-jl.yml | 59 -------------------- .github/workflows/lin-jl.yml | 61 --------------------- .github/workflows/mingw-cmake.yml | 91 ------------------------------- 7 files changed, 173 insertions(+), 211 deletions(-) create mode 100644 .github/workflows/julia-auto.yml create mode 100644 .github/workflows/julia-cmake.yml delete mode 100644 .github/workflows/lin-auto-jl.yml delete mode 100644 .github/workflows/lin-jl.yml delete mode 100644 .github/workflows/mingw-cmake.yml diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml index b75e75d639f..5be31f72300 100644 --- a/.github/workflows/autotools.yml +++ b/.github/workflows/autotools.yml @@ -83,4 +83,10 @@ jobs: with: build_mode: "production" + call-release-cmake-julia: + name: "Autotools Julia Workflows" + uses: ./.github/workflows/julia-auto.yml + with: + build_mode: "production" + \ No newline at end of file diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index c0077ce9207..4a52424c2b2 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -85,4 +85,10 @@ jobs: uses: ./.github/workflows/cygwin-cmake.yml with: build_mode: "Release" + + call-release-cmake-julia: + name: "CMake Julia Workflows" + uses: ./.github/workflows/julia-cmake.yml + with: + build_mode: "Release" \ No newline at end of file diff --git a/.github/workflows/julia-auto.yml b/.github/workflows/julia-auto.yml new file mode 100644 index 00000000000..a7dd2ab15b9 --- /dev/null +++ b/.github/workflows/julia-auto.yml @@ -0,0 +1,79 @@ +name: hdf5 dev autotools julia + +on: + workflow_call: + inputs: + build_mode: + description: "release vs. debug build" + required: true + type: string + +permissions: + contents: read + +jobs: + julia_build_and_test: + name: "julia ${{ inputs.build_mode }}" + runs-on: ubuntu-latest + steps: + - name: Get Sources + uses: actions/checkout@v4.1.1 + + - name: Install Dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install autoconf automake libtool libtool-bin libaec-dev + sudo apt-get install doxygen graphviz + sudo apt install -y zlib1g-dev libcurl4-openssl-dev libjpeg-dev wget curl bzip2 + sudo apt install -y m4 flex bison cmake libzip-dev openssl build-essential + + - name: Autotools Configure + shell: bash + run: | + sh ./autogen.sh + mkdir "${{ runner.workspace }}/build" + cd "${{ runner.workspace }}/build" + $GITHUB_WORKSPACE/configure \ + --enable-build-mode=${{ inputs.build_mode }} \ + --disable-fortran \ + --enable-shared \ + --disable-parallel \ + --prefix=/tmp + + - name: Autotools Build + shell: bash + run: | + make -j3 + working-directory: ${{ runner.workspace }}/build + + - name: Install HDF5 + shell: bash + run: | + make install + working-directory: ${{ runner.workspace }}/build + + - name: Install julia + uses: julia-actions/setup-julia@latest + with: + version: '1.6' + arch: 'x64' + + - name: Get julia hdf5 source + uses: actions/checkout@v4.1.1 + with: + repository: JuliaIO/HDF5.jl + path: . + + - name: Generate LocalPreferences + run: | + echo '[HDF5]' >> LocalPreferences.toml + echo 'libhdf5 = "/tmp/lib/libhdf5.so"' >> LocalPreferences.toml + echo 'libhdf5_hl = "/tmp/lib/libhdf5_hl.so"' >> LocalPreferences.toml + + - uses: julia-actions/julia-buildpkg@latest + + - name: Julia Run Tests + uses: julia-actions/julia-runtest@latest + env: + JULIA_DEBUG: Main diff --git a/.github/workflows/julia-cmake.yml b/.github/workflows/julia-cmake.yml new file mode 100644 index 00000000000..fb1de96f197 --- /dev/null +++ b/.github/workflows/julia-cmake.yml @@ -0,0 +1,82 @@ +name: hdf5 dev CMake julia + +on: + workflow_call: + inputs: + build_mode: + description: "release vs. debug build" + required: true + type: string + +permissions: + contents: read + +jobs: + julia_build_and_test: + name: "julia ${{ inputs.build_mode }}" + runs-on: ubuntu-latest + steps: + - name: Get Sources + uses: actions/checkout@v4.1.1 + + - name: Install Dependencies + shell: bash + run: | + sudo apt update + sudo apt-get install ninja-build doxygen graphviz + sudo apt install libssl3 libssl-dev libcurl4 libcurl4-openssl-dev + sudo apt install -y libaec-dev zlib1g-dev wget curl bzip2 flex bison cmake libzip-dev openssl build-essential + + - name: CMake Configure + shell: bash + run: | + mkdir "${{ runner.workspace }}/build" + cd "${{ runner.workspace }}/build" + cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_ENABLE_PARALLEL:BOOL=OFF \ + -DHDF5_BUILD_CPP_LIB:BOOL=OFF \ + -DLIBAEC_USE_LOCALCONTENT=OFF \ + -DZLIB_USE_LOCALCONTENT=OFF \ + -DHDF5_BUILD_FORTRAN:BOOL=OFF \ + -DHDF5_BUILD_JAVA:BOOL=OFF \ + -DCMAKE_INSTALL_PREFIX=/tmp \ + $GITHUB_WORKSPACE + + - name: CMake Build + shell: bash + run: | + cmake --build . --parallel 3 --config ${{ inputs.build_mode }} + working-directory: ${{ runner.workspace }}/build + + - name: Install HDF5 + shell: bash + run: | + cmake --install . + working-directory: ${{ runner.workspace }}/build + + - name: Install julia + uses: julia-actions/setup-julia@latest + with: + version: '1.6' + arch: 'x64' + + - name: Get julia hdf5 source + uses: actions/checkout@v4.1.1 + with: + repository: JuliaIO/HDF5.jl + path: . + + - name: Generate LocalPreferences + run: | + echo '[HDF5]' >> LocalPreferences.toml + echo 'libhdf5 = "/tmp/lib/libhdf5.so"' >> LocalPreferences.toml + echo 'libhdf5_hl = "/tmp/lib/libhdf5_hl.so"' >> LocalPreferences.toml + + - uses: julia-actions/julia-buildpkg@latest + + - name: Julia Run Tests + uses: julia-actions/julia-runtest@latest + env: + JULIA_DEBUG: Main diff --git a/.github/workflows/lin-auto-jl.yml b/.github/workflows/lin-auto-jl.yml deleted file mode 100644 index 2cc96ea77ac..00000000000 --- a/.github/workflows/lin-auto-jl.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: lin auto jl - -on: - workflow_dispatch: - push: - pull_request: - branches: [ develop ] - paths-ignore: - - '.github/CODEOWNERS' - - '.github/FUNDING.yml' - - 'doc/**' - - 'release_docs/**' - - 'ACKNOWLEDGEMENTS' - - 'COPYING**' - - '**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - Julia: - name: Julia - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - repository: HDFGroup/hdf5 - path: hdf5 - - name: Install HDF5 - run: | - cd hdf5 - ./autogen.sh - ./configure --prefix=/tmp - make -j - make install - - - uses: julia-actions/setup-julia@latest - with: - version: '1.6' - arch: 'x64' - - uses: actions/checkout@v4 - with: - repository: JuliaIO/HDF5.jl - path: . - - - name: Generate LocalPreferences - run: | - echo '[HDF5]' >> LocalPreferences.toml - echo 'libhdf5 = "/tmp/lib/libhdf5.so"' >> LocalPreferences.toml - echo 'libhdf5_hl = "/tmp/lib/libhdf5_hl.so"' >> LocalPreferences.toml - - - uses: julia-actions/julia-buildpkg@latest - - uses: julia-actions/julia-runtest@latest - env: - JULIA_DEBUG: Main diff --git a/.github/workflows/lin-jl.yml b/.github/workflows/lin-jl.yml deleted file mode 100644 index 23839a1b57c..00000000000 --- a/.github/workflows/lin-jl.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: lin jl - -on: - workflow_dispatch: - push: - pull_request: - branches: [ develop ] - paths-ignore: - - '.github/CODEOWNERS' - - '.github/FUNDING.yml' - - 'doc/**' - - 'release_docs/**' - - 'ACKNOWLEDGEMENTS' - - 'COPYING**' - - '**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - Julia: - name: Julia - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - repository: HDFGroup/hdf5 - path: hdf5 - - name: Install HDF5 - run: | - cd hdf5 - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=/tmp .. - make -j - make install - cd .. - - - uses: julia-actions/setup-julia@latest - with: - version: '1.6' - arch: 'x64' - - uses: actions/checkout@v4 - with: - repository: JuliaIO/HDF5.jl - path: . - - - name: Generate LocalPreferences - run: | - echo '[HDF5]' >> LocalPreferences.toml - echo 'libhdf5 = "/tmp/lib/libhdf5.so"' >> LocalPreferences.toml - echo 'libhdf5_hl = "/tmp/lib/libhdf5_hl.so"' >> LocalPreferences.toml - - - uses: julia-actions/julia-buildpkg@latest - - uses: julia-actions/julia-runtest@latest - env: - JULIA_DEBUG: Main diff --git a/.github/workflows/mingw-cmake.yml b/.github/workflows/mingw-cmake.yml deleted file mode 100644 index 47613189cd2..00000000000 --- a/.github/workflows/mingw-cmake.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: hdf5 dev CMake MinGW - -on: - workflow_call: - inputs: - build_mode: - description: "release vs. debug build" - required: true - type: string - shared: - description: "shared true/false" - required: true - type: string - netcdf: - description: "netcdf true/false" - required: true - type: string - -permissions: - contents: read - -jobs: - mingw_build_and_test: - name: "mingw-${{ inputs.build_mode }}-NC=${{ inputs.netcdf }}" - if: "!contains(github.event.head_commit.message, 'skip-ci')" - runs-on: ubuntu-latest - steps: - - name: Get Sources - uses: actions/checkout@v4.1.1 - - - name: Install Dependencies - shell: bash - run: | - sudo apt update - sudo apt-get install -y ninja-build libtirpc-dev graphviz - - - name: Install MinGW - uses: egor-tensin/setup-mingw@v2 - with: - platform: x64 - - - name: Install Doxygen - uses: ssciwr/doxygen-install@v1 - with: - version: "1.9.7" - - - name: CMake Configure - shell: bash - run: | - mkdir "${{ runner.workspace }}/build" - cd "${{ runner.workspace }}/build" - cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake \ - -G Ninja \ - --log-level=VERBOSE \ - -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ - -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/config/toolchain/mingw64.cmake \ - -DBUILD_SHARED_LIBS:BOOL=${{ inputs.shared }} \ - -DHDF4_BUILD_EXAMPLES:BOOL=ON \ - -DBUILD_JPEG_WITH_PIC:BOOL=ON \ - -DHDF4_ENABLE_NETCDF:BOOL=${{ inputs.netcdf }} \ - -DHDF4_BUILD_FORTRAN:BOOL=OFF \ - -DHDF4_BUILD_JAVA:BOOL=OFF \ - -DHDF4_BUILD_DOC:BOOL=ON \ - -DJPEG_USE_LOCALCONTENT:BOOL=OFF \ - -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF \ - -DZLIB_USE_LOCALCONTENT:BOOL=OFF \ - -DHDF4_PACK_EXAMPLES:BOOL=ON \ - -DHDF4_PACKAGE_EXTLIBS:BOOL=ON \ - $GITHUB_WORKSPACE - - - name: CMake Build - shell: bash - run: | - cmake --build . --parallel 3 --config ${{ inputs.build_mode }} - working-directory: ${{ runner.workspace }}/build - - - name: CMake Run Tests - shell: bash - run: | - ctest . --parallel 2 -C ${{ inputs.build_mode }} -V - if: false - - - name: CMake Package - shell: bash - run: | - cpack -C ${{ inputs.build_mode }} -V - working-directory: ${{ runner.workspace }}/build - - - name: List files in the space - run: | - ls -l ${{ runner.workspace }}/build From 98c19af7c8a57c58a2a97ce7b0b56c056a246860 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Fri, 29 Mar 2024 07:41:42 -0500 Subject: [PATCH 09/12] Fix range check in H5_addr_overlap (#4278) When the H5_addr_overlap macro was updated to use H5_RANGE_OVERLAP, it failed to take into account that H5_RANGE_OVERLAP expects the range to be inclusive. This lead to an assertion failure in H5MM_memcpy due to a memcpy operation on overlapping memory. This has been fixed by subtracting 1 from the calculated high bound values passed to H5_RANGE_OVERLAP --- src/H5private.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5private.h b/src/H5private.h index c795704ece6..65a99cfc6ed 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -479,7 +479,7 @@ (X) >= (Y)) #define H5_addr_cmp(X,Y) (H5_addr_eq((X), (Y)) ? 0 : \ (H5_addr_lt((X), (Y)) ? -1 : 1)) -#define H5_addr_overlap(O1,L1,O2,L2) H5_RANGE_OVERLAP(O1, ((O1)+(L1)), O2, ((O2)+(L2))) +#define H5_addr_overlap(O1,L1,O2,L2) H5_RANGE_OVERLAP(O1, ((O1)+(L1)-1), O2, ((O2)+(L2)-1)) /* clang-format on */ /* From 50d30bd4f01b71d02e37062ae8594884ff75ba93 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Fri, 29 Mar 2024 07:43:59 -0500 Subject: [PATCH 10/12] Fix potential buffer read overflows in H5PB_read (#4279) H5PB_read previously did not account for the fact that the size of the read it's performing could overflow the page buffer pointer, depending on the calculated offset for the read. This has been fixed by adjusting the size of the read if it's determined that it would overflow the page. --- release_docs/RELEASE.txt | 7 +++++++ src/H5PB.c | 11 ++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 28dbd2357be..d96e2f2a6f5 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -693,6 +693,13 @@ Bug Fixes since HDF5-1.14.0 release Library ------- + - Fixed potential buffer read overflows in H5PB_read + + H5PB_read previously did not account for the fact that the size of the + read it's performing could overflow the page buffer pointer, depending + on the calculated offset for the read. This has been fixed by adjusting + the size of the read if it's determined that it would overflow the page. + - Fixed CVE-2017-17507 This CVE was previously declared fixed, but later testing with a static diff --git a/src/H5PB.c b/src/H5PB.c index fc09cd56e96..69707d14cba 100644 --- a/src/H5PB.c +++ b/src/H5PB.c @@ -726,7 +726,7 @@ H5PB_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void * if (H5FD_MEM_DRAW == type) { last_page_addr = ((addr + size - 1) / page_buf->page_size) * page_buf->page_size; - /* How many pages does this write span */ + /* How many pages does this read span */ num_touched_pages = (last_page_addr / page_buf->page_size + 1) - (first_page_addr / page_buf->page_size); if (first_page_addr == last_page_addr) { @@ -835,6 +835,10 @@ H5PB_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void * offset = (0 == i ? addr - page_entry->addr : 0); buf_offset = (0 == i ? 0 : size - access_size); + /* Account for reads that would overflow a page */ + if (offset + access_size > page_buf->page_size) + access_size = page_buf->page_size - offset; + /* copy the requested data from the page into the input buffer */ H5MM_memcpy((uint8_t *)buf + buf_offset, (uint8_t *)page_entry->page_buf_ptr + offset, access_size); @@ -905,6 +909,11 @@ H5PB_read(H5F_shared_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void * /* Copy the requested data from the page into the input buffer */ offset = (0 == i ? addr - search_addr : 0); buf_offset = (0 == i ? 0 : size - access_size); + + /* Account for reads that would overflow a page */ + if (offset + access_size > page_buf->page_size) + access_size = page_buf->page_size - offset; + H5MM_memcpy((uint8_t *)buf + buf_offset, (uint8_t *)new_page_buf + offset, access_size); /* Create the new PB entry */ From 64f86353d1c888d26ed0e2104d44679fc2730b13 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Fri, 29 Mar 2024 11:34:46 -0500 Subject: [PATCH 11/12] Remove CRLF from NEWSLETTER.txt. (#4282) --- release_docs/NEWSLETTER.txt | 39 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/release_docs/NEWSLETTER.txt b/release_docs/NEWSLETTER.txt index 0a7637cf0bf..55b7dcd8203 100644 --- a/release_docs/NEWSLETTER.txt +++ b/release_docs/NEWSLETTER.txt @@ -1,19 +1,20 @@ -Release of HDF5 1.14.4 Library and Tools is now available from the HDF5 Releases page. - -This is a maintenance release with a few changes and updates: ----------------------------------------------------------------------------- - -* Added support for _Float16 16-bit floating-point datatype - - Support for the 16-bit floating-point _Float16 C type has been added to - HDF5. On platforms where this type is available, this can enable more - efficient storage of floating-point data when an application doesn't - need the precision of larger floating-point datatypes. It can also allow - for improved performance when converting between 16-bit floating-point - data and data of another HDF5 datatype. - - (GitHub #4065, #2154) - ----------------------------------------------------------------------------- -Please see the full release notes for detailed information regarding this release, -including a detailed list of changes. \ No newline at end of file +Release of HDF5 1.14.4 Library and Tools is now available from the HDF5 Releases page. + +This is a maintenance release with a few changes and updates: +---------------------------------------------------------------------------- + +* Added support for _Float16 16-bit floating-point datatype + + Support for the 16-bit floating-point _Float16 C type has been added to + HDF5. On platforms where this type is available, this can enable more + efficient storage of floating-point data when an application doesn't + need the precision of larger floating-point datatypes. It can also allow + for improved performance when converting between 16-bit floating-point + data and data of another HDF5 datatype. + + (GitHub #4065, #2154) + +---------------------------------------------------------------------------- +Please see the full release notes for detailed information regarding this release, +including a detailed list of changes. + From 855b95b34b12834f25469f65ce471542bee6fa6d Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Fri, 29 Mar 2024 11:46:31 -0700 Subject: [PATCH 12/12] Relaxed behavior of H5Pset_page_buffer_size() when opening files (#4280) This API call sets the size of a file's page buffer cache. This call was extremely strict about matching its parameters to the file strategy and page size used to create the file, requiring a separate open of the file to obtain these parameters. These requirements have been relaxed when using the fapl to open a previously-created file: * When opening a file that does not use the H5F_FSPACE_STRATEGY_PAGE strategy, the setting is ignored and the file will be opened, but without a page buffer cache. This was previously an error. * When opening a file that has a page size larger than the desired page buffer cache size, the page buffer cache size will be increased to the file's page size. This was previously an error. The behavior when creating a file using H5Pset_page_buffer_size() is unchanged. Fixes GitHub issue #3382 --- release_docs/RELEASE.txt | 23 ++++++++ src/H5Fint.c | 16 +++++- src/H5Ppublic.h | 11 +++- test/page_buffer.c | 113 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 159 insertions(+), 4 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index d96e2f2a6f5..9fd4be30ed1 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -368,6 +368,29 @@ New Features Library: -------- + - Relaxed behavior of H5Pset_page_buffer_size() when opening files + + This API call sets the size of a file's page buffer cache. This call + was extremely strict about matching its parameters to the file strategy + and page size used to create the file, requiring a separate open of the + file to obtain these parameters. + + These requirements have been relaxed when using the fapl to open + a previously-created file: + + * When opening a file that does not use the H5F_FSPACE_STRATEGY_PAGE + strategy, the setting is ignored and the file will be opened, but + without a page buffer cache. This was previously an error. + + * When opening a file that has a page size larger than the desired + page buffer cache size, the page buffer cache size will be increased + to the file's page size. This was previously an error. + + The behavior when creating a file using H5Pset_page_buffer_size() is + unchanged. + + Fixes GitHub issue #3382 + - Added support for _Float16 16-bit half-precision floating-point datatype Support for the _Float16 C datatype has been added on platforms where: diff --git a/src/H5Fint.c b/src/H5Fint.c index 3f5a1379834..45ded824a7e 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -2073,7 +2073,21 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if (H5F__super_read(file, a_plist, true) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock"); - /* Create the page buffer before initializing the superblock */ + /* Skip trying to create a page buffer if the file space strategy + * stored in the superblock isn't paged. + */ + if (shared->fs_strategy != H5F_FSPACE_STRATEGY_PAGE) + page_buf_size = 0; + + /* If the page buffer is enabled, the strategy is paged, and the size in + * the fapl is smaller than the file's page size, bump the page buffer + * size up to the file's page size. + */ + if (page_buf_size > 0 && shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE && + shared->fs_page_size > page_buf_size) + page_buf_size = shared->fs_page_size; + + /* Create the page buffer *after* reading the superblock */ if (page_buf_size) if (H5PB_create(shared, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer"); diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 804e7880280..2eb03c5a160 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -5748,7 +5748,16 @@ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t * If a non-zero page buffer size is set, and the file space strategy * is not set to paged or the page size for the file space strategy is * larger than the page buffer size, the subsequent call to H5Fcreate() - * or H5Fopen() using the \p plist_id will fail. + * using the \p plist_id will fail. + * + * \note As of HDF5 1.14.4, this property will be ignored when an existing + * file is being opened and the file space strategy stored in the + * file isn't paged. This was previously a failure. + * + * \note As of HDF5 1.14.4, if a file with a paged file space strategy is + * opened with a page size that is smaller than the file's page size, + * the page cache size will be rounded up to the file's page size. + * This was previously a failure. * * The function also allows setting the minimum percentage of pages for * metadata and raw data to prevent a certain type of data to evict hot diff --git a/test/page_buffer.c b/test/page_buffer.c index 8c977fedf44..05fa148a152 100644 --- a/test/page_buffer.c +++ b/test/page_buffer.c @@ -1650,6 +1650,114 @@ test_min_threshold(hid_t orig_fapl, const char *driver_name) } /* test_min_threshold */ +/*------------------------------------------------------------------------- + * Function: test_pb_fapl_tolerance_at_open() + * + * Purpose: Tests if the library tolerates setting fapl page buffer + * values via H5Pset_page_buffer_size() when opening a file + * that does not use page buffering or has a size smaller + * than the file's page size. + * + * As of HDF5 1.14.4, these should succeed. + * + * Return: 0 if test is successful + * 1 if test fails + * + *------------------------------------------------------------------------- + */ +static unsigned +test_pb_fapl_tolerance_at_open(void) +{ + const char *filename = "pb_fapl_tolerance.h5"; + hid_t fapl = H5I_INVALID_HID; + hid_t fcpl = H5I_INVALID_HID; + hid_t fid = H5I_INVALID_HID; + H5F_t *f = NULL; + + TESTING("if opening non-page-buffered files works w/ H5Pset_page_buffer_size()"); + + /* Create a file WITHOUT page buffering */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Set up page buffering values on a fapl */ + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_page_buffer_size(fapl, 512, 0, 0) < 0) + TEST_ERROR; + + /* Attempt to open non-page-buf file w/ page buf fapl. Should succeed, + * but without a page buffer. + */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR; + if (NULL == (f = (H5F_t *)H5VL_object(fid))) + TEST_ERROR; + if (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) + TEST_ERROR; + if (f->shared->page_buf != NULL) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Set up a fcpl with a page size that is larger than the fapl size */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) + TEST_ERROR; + if (H5Pset_file_space_page_size(fcpl, 4096) < 0) + TEST_ERROR; + + /* Create a file that uses page buffering with a larger page size */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Attempt to open page-buf file w/ fapl page buf size that is too small. + * Should succeed with a page buffer size that matches the file's page size. + */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR; + if (NULL == (f = (H5F_t *)H5VL_object(fid))) + TEST_ERROR; + if (f->shared->fs_strategy != H5F_FSPACE_STRATEGY_PAGE) + TEST_ERROR; + if (f->shared->page_buf == NULL) + TEST_ERROR; + if (f->shared->fs_page_size != 4096) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Shut down */ + if (H5Pclose(fcpl) < 0) + TEST_ERROR; + if (H5Pclose(fapl) < 0) + TEST_ERROR; + + HDremove(filename); + + PASSED(); + + return 0; + +error: + + H5E_BEGIN_TRY + { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fid); + } + H5E_END_TRY + + return 1; + +} /* test_pb_fapl_tolerance_at_open */ + /*------------------------------------------------------------------------- * Function: test_stats_collection() * @@ -2083,12 +2191,12 @@ main(void) SKIPPED(); puts("Skip page buffering test because paged aggregation is disabled for multi/split drivers"); exit(EXIT_SUCCESS); - } /* end if */ + } if ((fapl = h5_fileaccess()) < 0) { nerrors++; PUTS_ERROR("Can't get VFD-dependent fapl"); - } /* end if */ + } /* Push API context */ if (H5CX_push() < 0) @@ -2107,6 +2215,7 @@ main(void) nerrors += test_lru_processing(fapl, driver_name); nerrors += test_min_threshold(fapl, driver_name); nerrors += test_stats_collection(fapl, driver_name); + nerrors += test_pb_fapl_tolerance_at_open(); #endif /* H5_HAVE_PARALLEL */