Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle certain empty subfiling environment variables #4038

Merged
merged 25 commits into from
Mar 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
74573e7
Handle certain subfiling environment variables that are not set to an…
Feb 22, 2024
9659199
Fix all subfiling environment variables
Feb 26, 2024
69112cd
Committing clang-format changes
github-actions[bot] Feb 26, 2024
2a3d6f1
Update for Jordan's comments
Mar 8, 2024
110ab36
Fix conflict
Mar 8, 2024
88c5e94
Committing clang-format changes
github-actions[bot] Mar 8, 2024
08e6cf5
Passes all tests
Mar 8, 2024
8770bc6
Merge branch 'github_3978' of https://github.com/glennsong09/hdf5 int…
Mar 8, 2024
b768e0e
Make Jordan's changes
Mar 14, 2024
abf4d99
Committing clang-format changes
github-actions[bot] Mar 14, 2024
653f3d0
Test empty subfiling environment variables
Mar 15, 2024
bb26dfa
Committing clang-format changes
github-actions[bot] Mar 15, 2024
ea17485
Add test case for t_vfd
Mar 15, 2024
03d6798
Committing clang-format changes
github-actions[bot] Mar 15, 2024
4dd9238
Fixes
Mar 15, 2024
e49c046
Merge branch 'github_3978' of https://github.com/glennsong09/hdf5 int…
Mar 15, 2024
a596187
Committing clang-format changes
github-actions[bot] Mar 15, 2024
ee7bb78
Fixes
Mar 15, 2024
b3389aa
Merge branch 'github_3978' of https://github.com/glennsong09/hdf5 int…
Mar 15, 2024
dd5c687
Committing clang-format changes
github-actions[bot] Mar 15, 2024
62e71a6
Fix for Quincey/Jordan
Mar 19, 2024
10c6aaf
Committing clang-format changes
github-actions[bot] Mar 19, 2024
ccd9299
Remove saving env var code
Mar 19, 2024
8971492
Merge branch 'github_3978' of https://github.com/glennsong09/hdf5 int…
Mar 19, 2024
329ba12
Committing clang-format changes
github-actions[bot] Mar 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/H5FDsubfiling/H5FDioc.c
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ H5FD_ioc_init(void)

/* Check if IOC VFD has been loaded dynamically */
env_var = getenv(HDF5_DRIVER);
if (env_var && !strcmp(env_var, H5FD_IOC_NAME)) {
if (env_var && strlen(env_var) > 0 && !strcmp(env_var, H5FD_IOC_NAME)) {
int mpi_initialized = 0;
int provided = 0;

Expand Down Expand Up @@ -1498,7 +1498,8 @@ H5FD__ioc_del(const char *name, hid_t fapl)
/* TODO: No support for subfile directory prefix currently */
/* TODO: Possibly try loading config file prefix from file before deleting */
snprintf(tmp_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE,
prefix_env ? prefix_env : file_dirname, base_filename, (uint64_t)st.st_ino);
prefix_env && (strlen(prefix_env) > 0) ? prefix_env : file_dirname, base_filename,
(uint64_t)st.st_ino);

if (NULL == (config_file = fopen(tmp_filename, "r"))) {
if (ENOENT == errno) {
Expand Down
13 changes: 8 additions & 5 deletions src/H5FDsubfiling/H5subfiling_common.c
Original file line number Diff line number Diff line change
Expand Up @@ -777,7 +777,7 @@ init_subfiling(const char *base_filename, uint64_t file_id, H5FD_subfiling_param

/* Check if a prefix has been set for the configuration file name */
prefix_env = getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX);
if (prefix_env) {
if (prefix_env && (strlen(prefix_env) > 0)) {
if (NULL == (new_context->config_file_prefix = strdup(prefix_env)))
H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "couldn't copy config file prefix string");
}
Expand Down Expand Up @@ -851,7 +851,8 @@ init_subfiling(const char *base_filename, uint64_t file_id, H5FD_subfiling_param
char *env_value = NULL;

/* Check for a subfiling stripe size setting from the environment */
if ((env_value = getenv(H5FD_SUBFILING_STRIPE_SIZE))) {
env_value = getenv(H5FD_SUBFILING_STRIPE_SIZE);
if (env_value && (strlen(env_value) > 0)) {
long long stripe_size = -1;

errno = 0;
Expand Down Expand Up @@ -981,7 +982,8 @@ init_app_topology(int64_t sf_context_id, H5FD_subfiling_params_t *subfiling_conf
case SELECT_IOC_ONE_PER_NODE: {
if (comm_size > 1) {
/* Check for an IOC-per-node value set in the environment */
if ((env_value = getenv(H5FD_SUBFILING_IOC_PER_NODE))) {
env_value = getenv(H5FD_SUBFILING_IOC_PER_NODE);
if (env_value && (strlen(env_value) > 0)) {
errno = 0;
ioc_select_val = strtol(env_value, NULL, 0);
if ((ERANGE == errno)) {
Expand Down Expand Up @@ -1186,7 +1188,7 @@ get_ioc_selection_criteria_from_env(H5FD_subfiling_ioc_select_t *ioc_selection_t

*ioc_sel_info_str = NULL;

if (env_value) {
if (env_value && (strlen(env_value) > 0)) {
/*
* Parse I/O Concentrator selection strategy criteria as
* either a single value or two colon-separated values of
Expand Down Expand Up @@ -1821,7 +1823,8 @@ init_subfiling_context(subfiling_context_t *sf_context, const char *base_filenam
"couldn't allocate space for subfiling filename");

/* Check for a subfile name prefix setting in the environment */
if ((env_value = getenv(H5FD_SUBFILING_SUBFILE_PREFIX))) {
env_value = getenv(H5FD_SUBFILING_SUBFILE_PREFIX);
if (env_value && (strlen(env_value) > 0)) {
if (NULL == (sf_context->subfile_prefix = strdup(env_value)))
H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't copy subfile prefix value");
}
Expand Down
38 changes: 38 additions & 0 deletions testpar/t_subfiling_vfd.c
Original file line number Diff line number Diff line change
Expand Up @@ -3237,6 +3237,44 @@ main(int argc, char **argv)
SKIPPED();
#endif

if (MAINPROCESS)
printf("\nRe-running tests with environment variables set to the empty string\n");

HDsetenv("H5FD_SUBFILING_SUBFILE_PREFIX", "", 1);
HDsetenv("H5FD_SUBFILING_IOC_SELECTION_CRITERIA", "", 1);
HDsetenv("H5FD_SUBFILING_IOC_PER_NODE", "", 1);
HDsetenv("H5FD_SUBFILING_STRIPE_SIZE", "", 1);
HDsetenv("H5FD_SUBFILING_CONFIG_FILE_PREFIX", "", 1);

/* Grab values from environment variables if set */
parse_subfiling_env_vars();

/*
* Assume that we use the "one IOC per node" selection
* strategy by default, with a possibly modified
* number of IOCs per node value
*/
num_iocs_g = (ioc_per_node_g > 0) ? (int)ioc_per_node_g * num_nodes_g : num_nodes_g;
if (num_iocs_g > mpi_size)
num_iocs_g = mpi_size;

for (size_t i = 0; i < ARRAY_SIZE(tests); i++) {
if (MPI_SUCCESS == (mpi_code_g = MPI_Barrier(comm_g))) {
(*tests[i])();
}
else {
if (MAINPROCESS)
MESG("MPI_Barrier failed");
nerrors++;
}
}

HDunsetenv("H5FD_SUBFILING_SUBFILE_PREFIX");
HDunsetenv("H5FD_SUBFILING_IOC_SELECTION_CRITERIA");
HDunsetenv("H5FD_SUBFILING_IOC_PER_NODE");
HDunsetenv("H5FD_SUBFILING_STRIPE_SIZE");
HDunsetenv("H5FD_SUBFILING_CONFIG_FILE_PREFIX");

if (nerrors)
goto exit;

Expand Down
40 changes: 40 additions & 0 deletions testpar/t_vfd.c
Original file line number Diff line number Diff line change
Expand Up @@ -6250,6 +6250,46 @@ main(int argc, char **argv)

test_vector_io(mpi_rank, mpi_size);

#ifdef H5_HAVE_SUBFILING_VFD

if (mpi_rank == 0)
printf("\n --- TESTING SUBFILING VFD: environment variables set to empty --- \n");

HDsetenv("H5FD_SUBFILING_SUBFILE_PREFIX", "", 1);
HDsetenv("H5FD_SUBFILING_IOC_SELECTION_CRITERIA", "", 1);
HDsetenv("H5FD_SUBFILING_IOC_PER_NODE", "", 1);
HDsetenv("H5FD_SUBFILING_STRIPE_SIZE", "", 1);
HDsetenv("H5FD_SUBFILING_CONFIG_FILE_PREFIX", "", 1);

MPI_Barrier(comm);

if (mpi_rank == 0)
printf("\n --- TESTING MPIO VFD: selection I/O --- \n");

test_selection_io(mpi_rank, mpi_size);

if (mpi_rank == 0)
printf("\n --- TESTING MPIO VFD: vector I/O --- \n");

if (mpi_size < 2) {
if (mpi_rank == 0) {
printf(" Need at least 2 processes to run tests for vector I/O.");
SKIPPED();
}
printf("\n");
goto finish;
}

test_vector_io(mpi_rank, mpi_size);

HDunsetenv("H5FD_SUBFILING_SUBFILE_PREFIX");
HDunsetenv("H5FD_SUBFILING_IOC_SELECTION_CRITERIA");
HDunsetenv("H5FD_SUBFILING_IOC_PER_NODE");
HDunsetenv("H5FD_SUBFILING_STRIPE_SIZE");
HDunsetenv("H5FD_SUBFILING_CONFIG_FILE_PREFIX");

#endif

finish:
/* make sure all processes are finished before final report, cleanup
* and exit.
Expand Down