VOL: Cannot Create and Write an Attribute at File Create Time

[yzanhua]

VOL: Cannot Create and Write an Attribute at File Create Time

I am developing an HDF5 VOL and I can not create an attribute and write to it at file create time. This issue happens only for HDF5 1.13.3. Using HDF5 1.13.2 and before does not cause any problems.

---

How to reproduce:

Minimal VOL to reproduce the problem

A minimal VOL to reproduce the problem can be made by simply modifying the Passthru VOL provided in the HDF5 source code.

Only H5VL_pass_through_file_create needs to be modified. The modified H5VL_pass_through_file_create codes are provided below. Compared with the original H5VL_pass_through_file_create, some extra codes (surrounded by "begin extra codes" and "end extra codes") are added. These extra codes create a new attribute called __test_attr and write some random numbers to it. A segmentation fault (typo) An assertion failure will occur inside H5VLattr_write.

Click here to see the codes

static void *
H5VL_pass_through_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t dxpl_id, void **req)
{
    H5VL_pass_through_info_t *info;
    H5VL_pass_through_t      *file;
    hid_t                     under_fapl_id;
    void                     *under;

#ifdef ENABLE_PASSTHRU_LOGGING
    printf("------- PASS THROUGH VOL FILE Create\n");
#endif
    /* Get copy of our VOL info from FAPL */
    H5Pget_vol_info(fapl_id, (void **)&info);

    /* Make sure we have info about the underlying VOL to be used */
    if (!info)
        return NULL;

    /* Copy the FAPL */
    under_fapl_id = H5Pcopy(fapl_id);

    /* Set the VOL ID and info for the underlying FAPL */
    H5Pset_vol(under_fapl_id, info->under_vol_id, info->under_vol_info);

    /* Open the file with the underlying VOL connector */
    under = H5VLfile_create(name, flags, fcpl_id, under_fapl_id, dxpl_id, req);
    if (under) {
        file = H5VL_pass_through_new_obj(under, info->under_vol_id);
        {// begin extra codes
            int buf[5] = {70, 71, 72, 73, 74};  // some random numbers
            hsize_t len = 5;  // len of buffer
            H5VL_loc_params_t loc;
            loc.obj_type = H5I_FILE;
            loc.type     = H5VL_OBJECT_BY_SELF;
            hid_t asid = H5Screate_simple (1, &len, &len);
            void* ap = H5VLattr_create (under, &loc, info->under_vol_id, "__test_attr", 
                            H5T_STD_I32LE, asid, H5P_ATTRIBUTE_CREATE_DEFAULT,
                          H5P_ATTRIBUTE_ACCESS_DEFAULT, dxpl_id, NULL);
            H5VLattr_write (ap, info->under_vol_id, H5T_NATIVE_INT32, buf, dxpl_id, NULL);
            H5VLattr_close (ap, info->under_vol_id, dxpl_id, NULL);
            H5Sclose (asid);
        }// end extra codes

        /* Check for async request */
        if (req && *req)
            *req = H5VL_pass_through_new_obj(*req, info->under_vol_id);
    } /* end if */
    else
        file = NULL;

    /* Close underlying FAPL */
    H5Pclose(under_fapl_id);

    /* Release copy of our VOL info */
    H5VL_pass_through_info_free(info);

    return (void *)file;
} /* end H5VL_pass_through_file_create() */

Example user application

Click here to see test.c

#include <hdf5.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>

#define CHECK_ERR(A)                                                           \
  {                                                                            \
    if (A < 0) {                                                               \
      printf("Error at line %d: code %d\n", __LINE__, A);                      \
      goto err_out;                                                            \
    }                                                                          \
  }

int main(int argc, char **argv) {
  herr_t err = 0;
  const char *file_name = "test.h5";

  // VOL IDs
  hid_t modified_passthru_vol_id = -1, native_vol_id=-1;

  hid_t fid = -1;          // File ID
  hid_t faplid = -1;       // File access property List
  hid_t dxplid = -1;       // Data transfer property List

  H5VL_pass_through_info_t pass_through_info;

  // init MPI
  MPI_Init(&argc, &argv);

  // Get the id of modified passthru VOL
  modified_passthru_vol_id = H5VL_pass_through_register();
  CHECK_ERR(modified_passthru_vol_id);

  // Get the id of native VOL
  native_vol_id = H5VLpeek_connector_id_by_name ("native");

  // set the underlying VOL to native VOL
  pass_through_info.under_vol_id = native_vol_id;
  pass_through_info.under_vol_info = NULL;

  // create a faplid and set the underlying VOL
  faplid = H5Pcreate(H5P_FILE_ACCESS);
  CHECK_ERR(faplid);
  err = H5Pset_vol(faplid, modified_passthru_vol_id, &pass_through_info);
  CHECK_ERR(err);

  /* Create file using LOG VOL. */
  fid = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, faplid);
  CHECK_ERR(fid);

  /* Close everything */
err_out:;
  if (fid >= 0)
    H5Fclose(fid);
  if (faplid >= 0)
    H5Pclose(faplid);
  if (modified_passthru_vol_id >= 00)
    H5VLclose(modified_passthru_vol_id);

  MPI_Finalize();

  return 0;
}

Click here to see makefile

TARGET=test
HDF5_DIR=${HOME}/HDF5/1.13.3

all:
	mpicc -o ${TARGET} ${TARGET}.c -g -O2 -I${HDF5_DIR}/include -L${HDF5_DIR}/lib -lhdf5

run:
	LD_LIBRARY_PATH="${HDF5_DIR}/lib:${LD_LIBRARY_PATH}" \
	mpirun -np 1 ./${TARGET}

clean:
	rm -rf ${TARGET} test.h5 core.* *.h5

---

Output from gdb

Click here to see gdb output

#0  0x00007f7e2d2aea9f in raise ()
   from /lib64/libc.so.6
#1  0x00007f7e2d281e05 in abort ()
   from /lib64/libc.so.6
#2  0x00007f7e2d281cd9 in __assert_fail_base.cold.0 () from /lib64/libc.so.6
#3  0x00007f7e2d2a73f6 in __assert_fail ()
   from /lib64/libc.so.6
#4  0x00007f7e2e990e27 in H5T_patch_vlen_file (
    dt=<optimized out>, file=<optimized out>)
    at H5T.c:6241
#5  0x00007f7e2e65d6bb in H5A__write (
    attr=attr@entry=0x14b43e0,
    mem_type=0x14881e0,
    buf=buf@entry=0x7ffc589f6a90)
    at H5Aint.c:840
#6  0x00007f7e2ea59a59 in H5VL__native_attr_write (attr=0x14b43e0, dtype_id=216172782113783843,
    buf=0x7ffc589f6a90,
    dxpl_id=<optimized out>,
    req=<optimized out>)
    at H5VLnative_attr.c:231
#7  0x00007f7e2ea36fec in H5VL__attr_write (
    obj=obj@entry=0x14b43e0, cls=0x146d810,
    mem_type_id=mem_type_id@entry=216172782113783843, buf=buf@entry=0x7ffc589f6a90,
    dxpl_id=dxpl_id@entry=792633534417207304,
    req=req@entry=0x0) at H5VLcallback.c:1302
#8  0x00007f7e2ea3f2cd in H5VLattr_write (
    obj=obj@entry=0x14b43e0,
--Type <RET> for more, q to quit, c to continue without paging--
    216172782113783843, buf=buf@entry=0x7ffc589f6a90,
    dxpl_id=dxpl_id@entry=792633534417207304, req=req@entry=0x0)
    at H5VLcallback.c:1371
#9  0x00007f7e2ea66907 in H5VL_pass_through_file_create (name=<optimized out>,
    flags=<optimized out>, fcpl_id=<optimized out>, fapl_id=<optimized out>,
    dxpl_id=792633534417207304, req=0x0) at H5VLpassthru.c:1671
#10 0x00007f7e2ea388f4 in H5VL__file_create (cls=cls@entry=0x14b0220,
    name=name@entry=0x400c4a "test.h5", flags=flags@entry=19,
    fcpl_id=fcpl_id@entry=792633534417207300,
    fapl_id=fapl_id@entry=792633534417207315,
    dxpl_id=dxpl_id@entry=792633534417207304, req=0x0) at H5VLcallback.c:3570
#11 0x00007f7e2ea45596 in H5VL_file_create (
    connector_prop=connector_prop@entry=0x7ffc589f6bf0,
    name=name@entry=0x400c4a "test.h5", flags=flags@entry=19,
    fcpl_id=fcpl_id@entry=792633534417207300, fapl_id=792633534417207315,
    dxpl_id=792633534417207304, req=0x0) at H5VLcallback.c:3604
#12 0x00007f7e2e760d83 in H5F__create_api_common (
    filename=filename@entry=0x400c4a "test.h5", flags=19, flags@entry=2,
    fcpl_id=792633534417207300, fcpl_id@entry=0, fapl_id=<optimized out>,
    fapl_id@entry=792633534417207315, token_ptr=token_ptr@entry=0x0) at H5F.c:613
#13 0x00007f7e2e76363a in H5Fcreate (filename=filename@entry=0x400c4a "test.h5",
    flags=flags@entry=2, fcpl_id=fcpl_id@entry=0,
    fapl_id=fapl_id@entry=792633534417207315) at H5F.c:660
#14 0x0000000000400a12 in main (argc=<optimized out>, argv=<optimized out>)
    at test.c:48

---

Libraries Version.

1. MPICH 3.4.2

2. HDF5 1.13.3

   Configure command for HDF5:

   HDF5_VER=1.13.3
   
   ../hdf5-${HDF5_VER}/configure \
   --prefix=${LOCAL_HOME}/HDF5/${HDF5_VER} \
   --enable-parallel \
   --enable-build-mode=debug \
   CC=mpicc CXX=mpicxx

---

Extra Info

1. Seems that this issue only happens if we set --enable-build-mode=debug at HDF5 configure time. Setting --enable-build-mode=production seems not causing any issues but I am not 100% sure about this.
2. No similar issues for HDF5 1.13.2.

[yzanhua]

Update: It's suggested to use the H5VLXXX_lib_state calls when creating and writing an attribute at file create time. However, using these lib_state calls does not resolve the issue.

The following is the updated minimal VOL's H5VL_pass_through_file_create that uses lib_state calls but can still trigger the assertion failure.

click here to see modified H5VL_pass_through_file_create

static void *
H5VL_pass_through_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t dxpl_id,
                              void **req)
{
    H5VL_pass_through_info_t *info;
    H5VL_pass_through_t      *file;
    hid_t                     under_fapl_id;
    void                     *under;

#ifdef ENABLE_PASSTHRU_LOGGING
    printf("------- PASS THROUGH VOL FILE Create\n");
#endif

    /* Get copy of our VOL info from FAPL */
    H5Pget_vol_info(fapl_id, (void **)&info);

    /* Make sure we have info about the underlying VOL to be used */
    if (!info)
        return NULL;

    /* Copy the FAPL */
    under_fapl_id = H5Pcopy(fapl_id);

    /* Set the VOL ID and info for the underlying FAPL */
    H5Pset_vol(under_fapl_id, info->under_vol_id, info->under_vol_info);

    /* Open the file with the underlying VOL connector */
    under = H5VLfile_create(name, flags, fcpl_id, under_fapl_id, dxpl_id, req);
    if (under) {
        file = H5VL_pass_through_new_obj(under, info->under_vol_id);
        {// begin extra codes
            void *lib_state;
            H5VLretrieve_lib_state(&lib_state);
            H5VLstart_lib_state ();
            H5VLrestore_lib_state (lib_state);
            
            int buf[5] = {70, 71, 72, 73, 74};  // some random numbers
            hsize_t len = 5;
            H5VL_loc_params_t loc;
            loc.obj_type = H5I_FILE;
            loc.type     = H5VL_OBJECT_BY_SELF;
            hid_t asid = H5Screate_simple (1, &len, &len);
            void* ap = H5VLattr_create (under, &loc, info->under_vol_id, "__test_attr", 
                            H5T_STD_I32LE, asid, H5P_ATTRIBUTE_CREATE_DEFAULT,
                          H5P_ATTRIBUTE_ACCESS_DEFAULT, dxpl_id, NULL);
            H5VLattr_write (ap, info->under_vol_id, H5T_NATIVE_INT32, buf, dxpl_id, NULL);
            H5VLattr_close (ap, info->under_vol_id, dxpl_id, NULL);
            H5Sclose (asid);
            
            H5VLfinish_lib_state();
            H5VLrestore_lib_state(lib_state);
            H5VLfree_lib_state(lib_state);
        }// end extra codes

        /* Check for async request */
        if (req && *req)
            *req = H5VL_pass_through_new_obj(*req, info->under_vol_id);
    } /* end if */
    else
        file = NULL;

    /* Close underlying FAPL */
    H5Pclose(under_fapl_id);

    /* Release copy of our VOL info */
    H5VL_pass_through_info_free(info);

    return (void *)file;
} /* end H5VL_pass_through_file_create() */

[fortnern]

For now, passthrough VOL connectors should avoid doing anything with the file in the open and create callbacks except opening it. If connectors need to do anything else they should use the post open callback (H5VL_NATIVE_FILE_POST_OPEN op_type for the file "optional" callback), making sure to perform operations on the file only after passing the post open call down to the terminal connector. Note the post open callback is made for both file open and file create calls.

In future versions we may change things to allow manipulation of the file in create and open callbacks, and possibly remove the post open callback.

[fortnern]

We will make sure to document this limitation