From 4d4bb1d93f705be17d075f7fdceb00af636262f3 Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Mon, 15 May 2023 13:35:12 +0200 Subject: [PATCH 1/8] ROS3: (feature) Temporary security credentials - Implemented support for AWS temporary security credentials. For this kind of credentials also a session/security token should be included in the request by adding the x-amz-security-token header. --- release_docs/RELEASE.txt | 4 + src/H5FDros3.c | 268 ++++++++++++++++++++++++++++- src/H5FDros3.h | 22 +++ src/H5FDs3comms.c | 77 ++++++--- src/H5FDs3comms.h | 3 +- test/s3comms.c | 42 ++--- tools/lib/h5tools.c | 3 + tools/lib/h5tools.h | 1 + tools/lib/h5tools_utils.c | 78 ++++++--- tools/lib/h5tools_utils.h | 4 +- tools/libtest/h5tools_test_utils.c | 204 ++++++++++++++-------- tools/src/h5dump/h5dump.c | 8 +- tools/src/h5ls/h5ls.c | 8 +- tools/src/h5stat/h5stat.c | 11 +- 14 files changed, 581 insertions(+), 152 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 4f27dc53348..c875727a6bb 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -168,6 +168,10 @@ New Features order to avoid malloc/memcpy. Currently only used for type conversion with selection I/O. + - Added H5Pset_fapl_ros3_token() and H5Pget_fapl_ros3_token() API functions + to set or get the session/security token. This token is needed when + making use of AWS temporary security credentials. + Parallel Library: ----------------- diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 9a529d68a0e..26cf4ef2de7 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -50,6 +50,10 @@ */ static hid_t H5FD_ROS3_g = 0; +/* Session/security token property name + */ +#define ROS3_TOKEN_PROP_NAME "ros3_token_prop" + #if ROS3_STATS /* arbitrarily large value, such that any reasonable size read will be "less" @@ -236,6 +240,11 @@ static herr_t H5FD__ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing static herr_t H5FD__ros3_validate_config(const H5FD_ros3_fapl_t *fa); +static herr_t H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value); +static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size); +static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *value); +static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *value); + static const H5FD_class_t H5FD_ros3_g = { H5FD_CLASS_VERSION, /* struct version */ H5FD_ROS3_VALUE, /* value */ @@ -599,6 +608,240 @@ H5FD__ros3_fapl_free(void *_fa) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD__ros3_fapl_free() */ +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_ros3_token + * + * Purpose: Returns session/secutiry token of the ros3 file access + * property list though the function arguments. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) +{ + H5P_genplist_t *plist = NULL; + char token_src[H5FD_ROS3_MAX_SECRET_TOK_LEN] = {0}; + htri_t token_exists; + size_t tokenlen = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*s", fapl_id, token); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pget_fapl_ros3_token() called.\n"); +#endif + + if (size == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size cannot be zero.") + if (token_dst == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "token_dst is NULL") + + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5FD_ROS3 != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, token_src) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") + } + + /* Copy the token data out */ + tokenlen = HDstrlen(token_src); + if (size < tokenlen) { + tokenlen = size - 1; + } + H5MM_memcpy(token_dst, token_src, sizeof(char) * tokenlen); + token_dst[tokenlen] = '\0'; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_fapl_ros3_token() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_copy() + * + * Purpose: Create a copy of the token string. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value) +{ + char **value = (char **)&_value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD__ros3_str_token_copy() called.\n"); +#endif + + if (*value) + if (NULL == (*value = HDstrdup(*value))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy string property token") + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD__ros3_str_token_copy() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_cmp() + * + * Purpose: Compares two token strings with each other. + * + * Return: + * - Equivalent: 0 + * - Not Equivalent: non-zero value + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static int +H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size) +{ + char const *value1 = (char const *)_value1; + char const *value2 = (char const *)_value2; + int ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + if (value1) { + if (value2) + ret_value = HDstrcmp(value1, value2); + else + ret_value = 1; + } + else { + if (value2) + ret_value = -1; + else + ret_value = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_cmp */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_close() + * + * Purpose: Closes/frees the memory associated to the token string. + * Currently, it is an empty implementation since there no + * additional treatment needed for this property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_close(const char *name, size_t size, void *value) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_close */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_delete() + * + * Purpose: Deletes the property token from the property list and frees + * the memory associated to the token string. + * Currently, it is an empty implementation since there no + * additional treatment needed for this property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *value) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_delete */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_ros3_token() + * + * Purpose: Modify the file access property list to use the H5FD_ROS3 + * driver defined in this source file. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) +{ + H5P_genplist_t *plist = NULL; + htri_t token_exists; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*s", fapl_id, token); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pset_fapl_ros3_token() called.\n"); +#endif + + if (fapl_id == H5P_DEFAULT) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list") + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5FD_ROS3 != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if (HDstrlen(token) >= H5FD_ROS3_MAX_SECRET_TOK_LEN) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, + "specified token exceeds the internally specified maximum string length") + + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") + + if (token_exists) { + if (H5P_set(plist, ROS3_TOKEN_PROP_NAME, (void *)token) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + } + else { + if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char) * H5FD_ROS3_MAX_SECRET_TOK_LEN, + (void *)token, NULL, NULL, NULL, NULL, H5FD__ros3_str_token_delete, + H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, H5FD__ros3_str_token_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist") + } + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_ros3_token() */ + #if ROS3_STATS /*---------------------------------------------------------------------------- * @@ -693,7 +936,10 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) unsigned char signing_key[SHA256_DIGEST_LENGTH]; s3r_t *handle = NULL; H5FD_ros3_fapl_t fa; - H5FD_t *ret_value = NULL; + H5P_genplist_t *plist = NULL; + htri_t token_exists; + char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = {0}; + H5FD_t *ret_value = NULL; FUNC_ENTER_PACKAGE @@ -720,6 +966,16 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)") + /* Session/security token */ + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, NULL, "not a file access property list") + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "failed to check if property token exists in plist") + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, token) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get token value") + } + /* open file; procedure depends on whether or not the fapl instructs to * authenticate requests or not. */ @@ -736,11 +992,15 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) (const char *)fa.aws_region, (const char *)iso8601now)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key") - handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, - (const unsigned char *)signing_key); + if (token_exists) + handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, + (const unsigned char *)signing_key, (const char *)token); + else + handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, + (const unsigned char *)signing_key, ""); } else - handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL, NULL); if (handle == NULL) /* If we want to check CURL's say on the matter in a controlled diff --git a/src/H5FDros3.h b/src/H5FDros3.h index f84b1a44a6d..bee2807d69c 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -68,6 +68,13 @@ * * String: "Secret Access Key" associated with the ID and resource. * + * `secret_tok` (char[]) + * + * String: "Security/Session Token" associated with the ID and resource. + * + * This secret_tok is only needed when you are using temporary security + * credentials. + * ****************************************************************************/ #define H5FD_CURR_ROS3_FAPL_T_VERSION 1 @@ -75,6 +82,7 @@ #define H5FD_ROS3_MAX_REGION_LEN 32 #define H5FD_ROS3_MAX_SECRET_ID_LEN 128 #define H5FD_ROS3_MAX_SECRET_KEY_LEN 128 +#define H5FD_ROS3_MAX_SECRET_TOK_LEN 1024 typedef struct H5FD_ros3_fapl_t { int32_t version; @@ -104,6 +112,20 @@ H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out); */ H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa); +/** + * \ingroup FAPL + * + * \todo Add missing documentation + */ +H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token); + +/** + * \ingroup FAPL + * + * \todo Add missing documentation + */ +H5_DLL herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token); + #ifdef __cplusplus } #endif diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 628c55ae89e..d08e97788f5 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -1023,7 +1023,8 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle) *---------------------------------------------------------------------------- */ s3r_t * -H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key) +H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key, + const char *token) { size_t tmplen = 0; CURL *curlh = NULL; @@ -1057,13 +1058,15 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const handle->region = NULL; handle->secret_id = NULL; handle->signing_key = NULL; + handle->token = NULL; handle->httpverb = NULL; /************************************* * RECORD AUTHENTICATION INFORMATION * *************************************/ - if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL)) { + if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) || + (token != NULL)) { /* if one exists, all three must exist */ if (region == NULL || region[0] == '\0') @@ -1072,6 +1075,8 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "secret id cannot be null."); if (signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "signing key cannot be null."); + if (token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "token cannot be null."); /* copy strings */ tmplen = HDstrlen(region) + 1; @@ -1091,6 +1096,12 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy."); H5MM_memcpy(handle->signing_key, signing_key, tmplen); + + tmplen = HDstrlen(token) + 1; + handle->token = (char *)H5MM_malloc(sizeof(char) * tmplen); + if (handle->token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle token copy."); + H5MM_memcpy(handle->token, token, tmplen); } /* if authentication information provided */ /************************ @@ -1304,23 +1315,24 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) else { /* authenticate request */ - char authorization[512 + 1]; - /* 512 := approximate max length... - * 67 - * + 8 - * + 64 - * + 128 - * + 20 - * + 128 + char authorization[2048 + 1]; + /* 2048 := approximate max length... + * 67 + * + 8 + * + 64 + * + 128 + * + 20 + * + 128 + * + 1024 */ - char buffer1[512 + 1]; /* -> Canonical Request -> Signature */ - char buffer2[256 + 1]; /* -> String To Sign -> Credential */ + char buffer1[2048 + 1]; /* -> Canonical Request -> Signature */ + char buffer2[256 + 1]; /* -> String To Sign -> Credential */ char iso8601now[ISO8601_SIZE]; - char signed_headers[48 + 1]; + char signed_headers[48 + 1024 + 1]; /* should be large enough for nominal listing: - * "host;range;x-amz-content-sha256;x-amz-date" - * + '\0', with "range;" possibly absent + * "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token" + * + '\0', with "range;" and/or "x-amz-security-token" possibly absent */ /* zero start of strings */ @@ -1338,6 +1350,8 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null secret_id."); if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null signing_key."); + if (handle->token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null token."); if (handle->httpverb == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null httpverb."); if (handle->purl->host == NULL) @@ -1369,6 +1383,15 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list."); HDassert(headers->magic == S3COMMS_HRB_NODE_MAGIC); + if (HDstrlen((const char *)handle->token) > 0) { + if (FAIL == + H5FD_s3comms_hrb_node_set(&headers, "x-amz-security-token", (const char *)handle->token)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header") + if (headers == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list."); + HDassert(headers->magic == S3COMMS_HRB_NODE_MAGIC); + } + if (rangebytesstr != NULL) { if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "Range", rangebytesstr)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set range header") @@ -1388,8 +1411,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) /**** COMPUTE AUTHORIZATION ****/ /* buffer1 -> canonical request */ - if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512, signed_headers, 48, request)) + if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 2048, signed_headers, 48 + 1024, request)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad canonical request"); + } /* buffer2->string-to-sign */ if (FAIL == H5FD_s3comms_tostringtosign(buffer2, buffer1, iso8601now, handle->region)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad string-to-sign"); @@ -1403,9 +1427,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string"); - ret = HDsnprintf(authorization, 512, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", + ret = HDsnprintf(authorization, 2048, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2, signed_headers, buffer1); - if (ret <= 0 || ret >= 512) + if (ret <= 0 || ret >= 2048) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string"); /* append authorization header to http request buffer */ @@ -1460,7 +1484,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) if (p_status != CURLE_OK) { if (CURLE_OK != curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem getting response code") - HDfprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %d\n", p_status, httpcode); + HDfprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %ld\n", p_status, httpcode); HDfprintf(stderr, "%s\n", curl_easy_strerror(p_status)); HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "problem while performing request."); } @@ -1618,8 +1642,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c size_t sh_size = (size_t)_sh_size; size_t cr_len = 0; /* working length of canonical request str */ size_t sh_len = 0; /* working length of signed headers str */ - char tmpstr[256 + 1]; - tmpstr[256] = 0; /* terminating NULL */ + char tmpstr[1024 + 1]; + tmpstr[1024] = 0; /* terminating NULL */ /* "query params" refers to the optional element in the URL, e.g. * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J @@ -1652,6 +1676,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c (size_t)3); /* three newline chars */ if (cr_len >= cr_size) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request"); + /* TODO: compiler warning */ ret = HDsnprintf(canonical_request_dest, (cr_size - 1), "%s\n%s\n%s\n", http_request->verb, http_request->resource, query_params); @@ -1664,8 +1689,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC); - ret = HDsnprintf(tmpstr, 256, "%s:%s\n", node->lowername, node->value); - if (ret < 0 || ret >= 256) + ret = HDsnprintf(tmpstr, 1024, "%s:%s\n", node->lowername, node->value); + if (ret < 0 || ret >= 1024) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s", node->lowername, node->value); cr_len += HDstrlen(tmpstr); @@ -1673,8 +1698,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request"); HDstrcat(canonical_request_dest, tmpstr); - ret = HDsnprintf(tmpstr, 256, "%s;", node->lowername); - if (ret < 0 || ret >= 256) + ret = HDsnprintf(tmpstr, 1024, "%s;", node->lowername); + if (ret < 0 || ret >= 1024) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to append semicolon to lowername %s", node->lowername); sh_len += HDstrlen(tmpstr); diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h index 1b21ad7510d..47857f66fbe 100644 --- a/src/H5FDs3comms.h +++ b/src/H5FDs3comms.h @@ -478,6 +478,7 @@ typedef struct { char *region; char *secret_id; unsigned char *signing_key; + char *token; } s3r_t; #define S3COMMS_S3R_MAGIC 0x44d8d79 @@ -509,7 +510,7 @@ H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle); H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle); H5_DLL s3r_t *H5FD_s3comms_s3r_open(const char url[], const char region[], const char id[], - const unsigned char signing_key[]); + const unsigned char signing_key[], const char token[]); H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest); diff --git a/test/s3comms.c b/test/s3comms.c index 45978fb2be6..bdc49d86d65 100644 --- a/test/s3comms.c +++ b/test/s3comms.c @@ -384,6 +384,7 @@ static int s3_test_credentials_loaded = 0; static char s3_test_aws_region[16] = ""; static char s3_test_aws_access_key_id[64] = ""; static char s3_test_aws_secret_access_key[128] = ""; +static char s3_test_aws_security_token[1024] = ""; static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; static hbool_t s3_test_bucket_defined = FALSE; @@ -1788,7 +1789,7 @@ test_s3r_get_filesize(void) JSVERIFY(0, H5FD_s3comms_s3r_get_filesize(NULL), "filesize of the null handle should be 0") - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL) JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) @@ -1898,14 +1899,14 @@ test_s3r_open(void) /* attempt anonymously */ - handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); /* attempt with authentication */ - handle = - H5FD_s3comms_s3r_open(url_missing, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_missing, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /************************* @@ -1914,7 +1915,7 @@ test_s3r_open(void) #if S3_TEST_RUN_TIMEOUT HDprintf("Opening on inactive port may hang for a minute; waiting for timeout\n"); - handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); #endif @@ -1924,20 +1925,21 @@ test_s3r_open(void) /* anonymous access on restricted file */ - handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); /* passed in a bad ID */ - handle = H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID", - (const unsigned char *)signing_key); + handle = + H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID", + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /* using an invalid signing key */ - handle = - H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)EMPTY_SHA256); + handle = H5FD_s3comms_s3r_open( + url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /******************************* @@ -1946,7 +1948,7 @@ test_s3r_open(void) /* anonymous */ - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL); JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), "did not get expected filesize") JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1954,9 +1956,9 @@ test_s3r_open(void) /* using authentication on anonymously-accessible file? */ - handle = - H5FD_s3comms_s3r_open(url_raven, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_raven, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle == NULL); JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1964,9 +1966,9 @@ test_s3r_open(void) /* authenticating */ - handle = - H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle == NULL); JSVERIFY(5458199, H5FD_s3comms_s3r_get_filesize(handle), NULL) JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -2055,7 +2057,7 @@ test_s3r_read(void) /* open file */ - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL) JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 189aafdab63..2219ad6e467 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -568,6 +568,9 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info) H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid"); if (H5Pset_fapl_ros3(fapl_id, (const H5FD_ros3_fapl_t *)vfd_info->info) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed"); + + if (vfd_info->token && H5Pset_fapl_ros3_token(fapl_id, vfd_info->token) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3_token() failed"); #else H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled"); #endif diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index 753a83be480..bb6b7e3b693 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -569,6 +569,7 @@ typedef struct h5tools_vfd_info_t { /* Pointer to information to be passed to the driver for its setup */ const void *info; + const char *token; /* Field specifying either the driver's name or value (ID) */ union { diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index 8a9d8060441..3da54973749 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -1028,9 +1028,10 @@ h5tools_getenv_update_hyperslab_bufsize(void) *---------------------------------------------------------------------------- */ herr_t -h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t *fapl_config_out) +h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t *fapl_config_out, + char *token_out) { - const char *ccred[3]; + const char *ccred[4]; unsigned nelems = 0; char *s3cred_src = NULL; char **s3cred = NULL; @@ -1041,14 +1042,20 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t H5TOOLS_GOTO_ERROR(FAIL, "failed to parse S3 VFD info tuple"); /* Sanity-check tuple count */ - if (nelems != 3) + if (nelems != 3 && nelems != 4) H5TOOLS_GOTO_ERROR(FAIL, "invalid S3 VFD credentials"); ccred[0] = (const char *)s3cred[0]; ccred[1] = (const char *)s3cred[1]; ccred[2] = (const char *)s3cred[2]; + if (nelems == 3) { + ccred[3] = ""; + } + else { + ccred[3] = (const char *)s3cred[3]; + } - if (0 == h5tools_populate_ros3_fapl(fapl_config_out, ccred)) + if (0 == h5tools_populate_ros3_fapl(fapl_config_out, token_out, ccred)) H5TOOLS_GOTO_ERROR(FAIL, "failed to populate S3 VFD FAPL config"); done: @@ -1093,29 +1100,30 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t * * NULL fapl pointer: (NULL, {...} ) * * Warning: In all cases below, fapl will be set as "default" * before error occurs. - * * NULL value strings: (&fa, {NULL?, NULL? NULL?, ...}) + * * NULL value strings: (&fa, {NULL?, NULL? NULL?, NULL?, ...}) * * Incomplete fapl info: - * * empty region, non-empty id, key either way - * * (&fa, {"", "...", "?"}) - * * empty id, non-empty region, key either way - * * (&fa, {"...", "", "?"}) - * * "non-empty key and either id or region empty - * * (&fa, {"", "", "...") - * * (&fa, {"", "...", "...") - * * (&fa, {"...", "", "...") + * * empty region, non-empty id, key either way, token either way + * * (&fa, token, {"", "...", "?", "?"}) + * * empty id, non-empty region, key either way, token either way + * * (&fa, token, {"...", "", "?", "?"}) + * * "non-empty key, token either way and either id or region empty + * * (&fa, token, {"", "", "...", "?") + * * (&fa, token, {"", "...", "...", "?") + * * (&fa, token, {"...", "", "...", "?") * * Any string would overflow allowed space in fapl definition. * or * 1 (success) * * Sets components in fapl_t pointer, copying strings as appropriate. * * "Default" fapl (valid version, authenticate->False, empty strings) * * `values` pointer is NULL - * * (&fa, NULL) - * * first three strings in `values` are empty ("") - * * (&fa, {"", "", "", ...} + * * (&fa, token, NULL) + * * first four strings in `values` are empty ("") + * * (&fa, token, {"", "", "", "", ...}) * * Authenticating fapl - * * region, id, and optional key provided - * * (&fa, {"...", "...", ""}) - * * (&fa, {"...", "...", "..."}) + * * region, id, optional key and option session token provided + * * (&fa, token, {"...", "...", "", ""}) + * * (&fa, token, {"...", "...", "...", ""}) + * * (&fa, token, {"...", "...", "...", "..."}) * * Programmer: Jacob Smith * 2017-11-13 @@ -1123,7 +1131,7 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t *---------------------------------------------------------------------------- */ int -h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) +h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **values) { int show_progress = 0; /* set to 1 for debugging */ int ret_value = 1; /* 1 for success, 0 for failure */ @@ -1141,6 +1149,14 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) goto done; } + if (token == NULL) { + if (show_progress) { + HDprintf(" ERROR: null pointer to token\n"); + } + ret_value = 0; + goto done; + } + if (show_progress) { HDprintf(" preset fapl with default values\n"); } @@ -1149,6 +1165,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) *(fa->aws_region) = '\0'; *(fa->secret_id) = '\0'; *(fa->secret_key) = '\0'; + *token = '\0'; /* sanity-check supplied values */ @@ -1174,6 +1191,13 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) ret_value = 0; goto done; } + if (values[3] == NULL) { + if (show_progress) { + HDprintf(" ERROR: token value cannot be NULL\n"); + } + ret_value = 0; + goto done; + } /* if region and ID are supplied (key optional), write to fapl... * fail if value would overflow @@ -1215,12 +1239,24 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) HDprintf(" secret_key set\n"); } + if (HDstrlen(values[3]) > H5FD_ROS3_MAX_SECRET_TOK_LEN) { + if (show_progress) { + HDprintf(" ERROR: token value too long\n"); + } + ret_value = 0; + goto done; + } + HDmemcpy(token, values[3], (HDstrlen(values[3]) + 1)); + if (show_progress) { + HDprintf(" token set\n"); + } + fa->authenticate = TRUE; if (show_progress) { HDprintf(" set to authenticate\n"); } } - else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0') { + else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0' || *values[3] != '\0') { if (show_progress) { HDprintf(" ERROR: invalid assortment of empty/non-empty values\n"); } diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index a4fde5c8e2d..f1c0e3328c5 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -129,8 +129,8 @@ H5TOOLS_DLL void h5tools_setstatus(int d_status); H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void); #ifdef H5_HAVE_ROS3_VFD H5TOOLS_DLL herr_t h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, - H5FD_ros3_fapl_t *fapl_config_out); -H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values); + H5FD_ros3_fapl_t *fapl_config_out, char *token); +H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **values); #endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS H5TOOLS_DLL herr_t h5tools_parse_hdfs_fapl_tuple(const char *tuple_str, int delim, diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c index f060e2c63c3..add84a52107 100644 --- a/tools/libtest/h5tools_test_utils.c +++ b/tools/libtest/h5tools_test_utils.c @@ -632,116 +632,142 @@ test_populate_ros3_fa(void) /* NULL fapl config pointer fails */ { - const char *values[] = {"x", "y", "z"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "a"; + const char *values[] = {"x", "y", "z"}; if (show_progress) { HDprintf("NULL fapl pointer\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(NULL, values), "fapl pointer cannot be null") + JSVERIFY(0, h5tools_populate_ros3_fapl(NULL, token, values), "fapl pointer cannot be null") + } + + /* NULL token pointer fails + */ + { + H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; + const char *values[] = {"x", "y", "z"}; + + if (show_progress) { + HDprintf("NULL token pointer\n"); + } + + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, NULL, values), "token pointer cannot be null") } /* NULL values pointer yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; + H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "x"; if (show_progress) { HDprintf("NULL values pointer\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, NULL), "NULL values pointer yields \"default\" fapl") + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, NULL), + "NULL values pointer yields \"default\" fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* all-empty values * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; - const char *values[] = {"", "", ""}; + H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "x"; + const char *values[] = {"", "", "", ""}; if (show_progress) { HDprintf("all empty values\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "empty values yields \"default\" fapl") + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "empty values yields \"default\" fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* successfully set fapl with values * excess value is ignored */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", "z", "a"}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "y", "z", "a", "b"}; if (show_progress) { HDprintf("successful full set\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "four values") + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "four values") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(TRUE, fa.authenticate, NULL) JSVERIFY_STR("x", fa.aws_region, NULL) JSVERIFY_STR("y", fa.secret_id, NULL) JSVERIFY_STR("z", fa.secret_key, NULL) + JSVERIFY_STR("a", token, NULL) } /* NULL region * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {NULL, "y", "z", NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {NULL, "y", "z", ""}; if (show_progress) { HDprintf("NULL region\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* empty region * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"", "y", "z", NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"", "y", "z", ""}; if (show_progress) { HDprintf("empty region; non-empty id, key\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* region overflow * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"somewhere over the rainbow not too high " - "there is another rainbow bounding some darkened sky", - "y", "z"}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"somewhere over the rainbow not too high " + "there is another rainbow bounding some darkened sky", + "y", "z", ""}; if (show_progress) { HDprintf("region overflow\n"); @@ -749,69 +775,75 @@ test_populate_ros3_fa(void) HDassert(HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN); - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* NULL id * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", NULL, "z", NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", NULL, "z", ""}; if (show_progress) { HDprintf("NULL id\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* empty id (non-empty region, key) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "", "z", NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "", "z", ""}; if (show_progress) { HDprintf("empty id; non-empty region and key\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* id overflow * partial set: region */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "Why is it necessary to solve the problem? " - "What benefits will you receive by solving the problem? " - "What is the unknown? " - "What is it you don't yet understand? " - "What is the information you have? " - "What isn't the problem? " - "Is the information insufficient, redundant, or contradictory? " - "Should you draw a diagram or figure of the problem? " - "What are the boundaries of the problem? " - "Can you separate the various parts of the problem?", - "z"}; + "What benefits will you receive by solving the problem? " + "What is the unknown? " + "What is it you don't yet understand? " + "What is the information you have? " + "What isn't the problem? " + "Is the information insufficient, redundant, or contradictory? " + "Should you draw a diagram or figure of the problem? " + "What are the boundaries of the problem? " + "Can you separate the various parts of the problem?", + "z", ""}; if (show_progress) { HDprintf("id overflow\n"); @@ -819,106 +851,138 @@ test_populate_ros3_fa(void) HDassert(HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN); - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("x", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* NULL key * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", NULL, NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "y", NULL, ""}; + + if (show_progress) { + HDprintf("NULL key\n"); + } + + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) + JSVERIFY(FALSE, fa.authenticate, NULL) + JSVERIFY_STR("", fa.aws_region, NULL) + JSVERIFY_STR("", fa.secret_id, NULL) + JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) + } + + /* NULL token + * yields default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "y", "z", NULL}; if (show_progress) { HDprintf("NULL key\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill token") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* empty key (non-empty region, id) * yields authenticating fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", "", NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "y", "", ""}; if (show_progress) { HDprintf("empty key; non-empty region and id\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(TRUE, fa.authenticate, NULL) JSVERIFY_STR("x", fa.aws_region, NULL) JSVERIFY_STR("y", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* empty key, region (non-empty id) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"", "y", "", NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"", "y", "", ""}; if (show_progress) { HDprintf("empty key and region; non-empty id\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* empty key, id (non-empty region) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "", "", NULL}; + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "", "", ""}; if (show_progress) { HDprintf("empty key and id; non-empty region\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("", fa.aws_region, NULL) JSVERIFY_STR("", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* key overflow * partial set: region, id */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; + const char *values[] = {"x", "y", "Why is it necessary to solve the problem? " - "What benefits will you receive by solving the problem? " - "What is the unknown? " - "What is it you don't yet understand? " - "What is the information you have? " - "What isn't the problem? " - "Is the information insufficient, redundant, or contradictory? " - "Should you draw a diagram or figure of the problem? " - "What are the boundaries of the problem? " - "Can you separate the various parts of the problem?"}; + "What benefits will you receive by solving the problem? " + "What is the unknown? " + "What is it you don't yet understand? " + "What is the information you have? " + "What isn't the problem? " + "Is the information insufficient, redundant, or contradictory? " + "Should you draw a diagram or figure of the problem? " + "What are the boundaries of the problem? " + "Can you separate the various parts of the problem?", + ""}; if (show_progress) { HDprintf("key overflow\n"); @@ -926,21 +990,23 @@ test_populate_ros3_fa(void) HDassert(HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN); - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) JSVERIFY(FALSE, fa.authenticate, NULL) JSVERIFY_STR("x", fa.aws_region, NULL) JSVERIFY_STR("y", fa.secret_id, NULL) JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY_STR("", token, NULL) } /* use case */ { - H5FD_ros3_fapl_t fa = {0, 0, "", "", ""}; - const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA", - "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"}; - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "unable to set use case") + H5FD_ros3_fapl_t fa = {0, 0, "", "", ""}; + const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = ""; + const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA", + "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+", ""}; + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "unable to set use case") JSVERIFY(1, fa.version, "version check") JSVERIFY(1, fa.authenticate, "should authenticate") } @@ -1115,6 +1181,7 @@ test_set_configured_fapl(void) }; /* testcases `cases` array */ unsigned int i; + char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = ""; #ifdef H5_HAVE_ROS3_VFD n_cases += 3; @@ -1156,6 +1223,7 @@ test_set_configured_fapl(void) vfd_info.type = VFD_BY_NAME; vfd_info.info = C.conf_fa; vfd_info.u.name = C.vfdname; + vfd_info.token = token; result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); if (C.expected == 0) { JSVERIFY(result, H5I_INVALID_HID, C.message) diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index 2a5eeb70b2d..dc398d8c2fd 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -39,7 +39,8 @@ static H5FD_ros3_fapl_t ros3_fa_g = { "", /* Access Key ID */ "", /* Secret Access Key */ }; -#endif /* H5_HAVE_ROS3_VFD */ +static char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ +#endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS /* "Default" HDFS configuration */ @@ -1147,7 +1148,7 @@ parse_command_line(int argc, const char *const *argv) case '$': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa_g) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa_g, token) < 0) { error_msg("failed to parse S3 VFD credential info\n"); usage(h5tools_getprogname()); free_handler(hand, argc); @@ -1156,7 +1157,8 @@ parse_command_line(int argc, const char *const *argv) goto done; } - vfd_info_g.info = &ros3_fa_g; + vfd_info_g.info = &ros3_fa_g; + vfd_info_g.token = token; #else error_msg("Read-Only S3 VFD not enabled.\n"); h5tools_setstatus(EXIT_FAILURE); diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index 5d9e184571d..ee92ab6e884 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -2663,7 +2663,8 @@ main(int argc, char *argv[]) "", /* Access Key ID */ "", /* Secret Access Key */ }; -#endif /* H5_HAVE_ROS3_VFD */ + char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ +#endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS /* "Default" HDFS configuration */ @@ -2847,13 +2848,14 @@ main(int argc, char *argv[]) } start++; - if (h5tools_parse_ros3_fapl_tuple(start, ',', &ros3_fa) < 0) { + if (h5tools_parse_ros3_fapl_tuple(start, ',', &ros3_fa, token) < 0) { HDfprintf(rawerrorstream, "Error: failed to parse S3 VFD credential info\n\n"); usage(); leave(EXIT_FAILURE); } - vfd_info.info = &ros3_fa; + vfd_info.info = &ros3_fa; + vfd_info.token = token; #else HDfprintf(rawerrorstream, "Error: Read-Only S3 VFD is not enabled\n\n"); usage(); diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 05e0be82a4b..58dbf216ee4 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -126,7 +126,8 @@ static H5FD_ros3_fapl_t ros3_fa = { "", /* Access Key ID */ "", /* Secret Access Key */ }; -#endif /* H5_HAVE_ROS3_VFD */ +static char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ +#endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS /* "Default" HDFS configuration */ @@ -963,7 +964,7 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re case 'w': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa, token) < 0) { error_msg("failed to parse S3 VFD credential info\n"); goto error; } @@ -1684,8 +1685,10 @@ main(int argc, char *argv[]) vfd_info.u.name = drivername; #ifdef H5_HAVE_ROS3_VFD - if (!HDstrcmp(drivername, drivernames[ROS3_VFD_IDX])) - vfd_info.info = &ros3_fa; + if (!HDstrcmp(drivername, drivernames[ROS3_VFD_IDX])) { + vfd_info.info = &ros3_fa; + vfd_info.token = token; + } #endif #ifdef H5_HAVE_LIBHDFS if (!HDstrcmp(drivername, drivernames[HDFS_VFD_IDX])) From f263a4a6170f247e664ac417e1b220c529be9f6c Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Mon, 5 Jun 2023 13:38:31 +0200 Subject: [PATCH 2/8] ROS3: (feature+fix) Temporary security credentials - comments - Free the allocated arrays. - For large arrays make use of dynamic memory allocation. - Added doxygen-style inline documentation. - Improved the existing inline documentation. - Fixed typos. --- src/H5FDros3.c | 11 ++--- src/H5FDros3.h | 102 ++++++++++++++++++++++++++++++++++++++++------ src/H5FDs3comms.c | 42 ++++++++++++++----- 3 files changed, 127 insertions(+), 28 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 26cf4ef2de7..b365acba86b 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -611,7 +611,7 @@ H5FD__ros3_fapl_free(void *_fa) /*------------------------------------------------------------------------- * Function: H5Pget_fapl_ros3_token * - * Purpose: Returns session/secutiry token of the ros3 file access + * Purpose: Returns session/security token of the ros3 file access * property list though the function arguments. * * Return: Success: Non-negative @@ -657,7 +657,7 @@ H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) /* Copy the token data out */ tokenlen = HDstrlen(token_src); - if (size < tokenlen) { + if (size <= tokenlen) { tokenlen = size - 1; } H5MM_memcpy(token_dst, token_src, sizeof(char) * tokenlen); @@ -791,7 +791,8 @@ H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void * * Function: H5Pset_fapl_ros3_token() * * Purpose: Modify the file access property list to use the H5FD_ROS3 - * driver defined in this source file. + * driver defined in this source file by adding or + * modifying the session/security token property. * * Return: SUCCEED/FAIL * @@ -820,7 +821,7 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list") if (H5FD_ROS3 != H5P_peek_driver(plist)) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") - if (HDstrlen(token) >= H5FD_ROS3_MAX_SECRET_TOK_LEN) + if (HDstrlen(token) > H5FD_ROS3_MAX_SECRET_TOK_LEN) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "specified token exceeds the internally specified maximum string length") @@ -832,7 +833,7 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") } else { - if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char) * H5FD_ROS3_MAX_SECRET_TOK_LEN, + if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1), (void *)token, NULL, NULL, NULL, NULL, H5FD__ros3_str_token_delete, H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, H5FD__ros3_str_token_close) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist") diff --git a/src/H5FDros3.h b/src/H5FDros3.h index bee2807d69c..767b178fb6e 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -68,22 +68,65 @@ * * String: "Secret Access Key" associated with the ID and resource. * - * `secret_tok` (char[]) - * - * String: "Security/Session Token" associated with the ID and resource. - * - * This secret_tok is only needed when you are using temporary security - * credentials. - * ****************************************************************************/ +/** + * \def H5FD_CURR_ROS3_FAPL_T_VERSION + * The version number of the H5FD_ros3_fapl_t configuration + * structure for the $H5FD_ROS3 driver. + */ #define H5FD_CURR_ROS3_FAPL_T_VERSION 1 -#define H5FD_ROS3_MAX_REGION_LEN 32 -#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +/** + * \def H5FD_ROS3_MAX_REGION_LEN + * Maximum string length for specifying the region of the S3 bucket. + */ +#define H5FD_ROS3_MAX_REGION_LEN 32 +/** + * \def H5FD_ROS3_MAX_SECRET_ID_LEN + * Maximum string length for specifying the security ID. + */ +#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +/** + * \def H5FD_ROS3_MAX_SECRET_KEY_LEN + * Maximum string length for specifying the security key. + */ #define H5FD_ROS3_MAX_SECRET_KEY_LEN 128 +/** + * \def H5FD_ROS3_MAX_SECRET_TOK_LEN + * Maximum string length for specifying the session/security token. + */ #define H5FD_ROS3_MAX_SECRET_TOK_LEN 1024 +/** + *\struct H5FD_ros3_fapl_t + * \brief Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3(). + * + * \details H5FD_ros_fapl_t is a public structure that is used to pass + * configuration data to the #H5FD_ROS3 driver via a File Access + * Property List. A pointer to an instance of this structure is + * a parameter to H5Pset_fapl_ros3() and H5Pget_fapl_ros3(). + * + * \var int32_t H5FD_ros3_fapl_t::version + * Version number of the H5FD_ros3_fapl_t structure. Any instance passed + * to H5Pset_fapl_ros3() / H5Pget_fapl_ros3() must have a recognized version + * number or an error will be raised. Currently, this field should be set + * to #H5FD_CURR_ROS3_FAPL_T_VERSION. + * + * \var hbool_t H5FD_ros3_fapl_t::authenticate + * A boolean which specifies if security credentials should be used for + * accessing a S3 bucket. + * + * \var char H5FD_ros3_fapl_t::aws_region[H5FD_ROS3_MAX_REGION_LEN + 1] + * A string which specifies the AWS region of the S3 bucket. + * + * \var char H5FD_ros3_fapl_t::secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1] + * A string which specified the security ID. + * + * \var char H5FD_ros3_fapl_t::secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1] + * A string which specified the security key. + * + */ typedef struct H5FD_ros3_fapl_t { int32_t version; hbool_t authenticate; @@ -96,33 +139,66 @@ typedef struct H5FD_ros3_fapl_t { extern "C" { #endif +/** + * \brief Internal routine to initialize #H5FD_ROS3 driver. Not meant to be + * called directly by an HDF5 application. + */ H5_DLL hid_t H5FD_ros3_init(void); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Queries a File Access Property List for #H5FD_ROS3 file driver properties. + * + * \fapl_id + * \param[out] fa_out Pointer to #H5FD_ROS3 driver configuration structure. + * \returns \herr_t */ H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver. + * + * \fapl_id + * \param[in] fa Pointer to #H5FD_ROS3 driver configuration structure. + * \returns \herr_t */ H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Queries a File Access Property List for #H5FD_ROS3 file driver session/security + * token. + * + * \fapl_id + * \param[in] size Size of the provided char array for storing the session/security token. + * \param[out] token Session/security token. + * \returns \herr_t + * + * \since 1.14.2 */ H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver + * by adding the specified session/security token. + * + * \fapl_id + * \param[in] token Session/security token. + * \returns \herr_t + * + * \details H5Pset_fapy_ros3_token() modifies the File Access Property List to use the + * #H5FD_ROS3 driver by adding or updating the session/security token. + * + * The session token is only needed when you want to access a S3 bucket + * using temporary security credentials. + * + * \since 1.14.2 */ H5_DLL herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token); diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index d08e97788f5..5ee13f01324 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -42,6 +42,7 @@ #include "H5Eprivate.h" /* error handling */ #include "H5MMprivate.h" /* memory management */ #include "H5FDs3comms.h" /* S3 Communications */ +#include "H5FDros3.h" /* ros3 file driver */ /****************/ /* Local Macros */ @@ -793,6 +794,7 @@ H5FD_s3comms_s3r_close(s3r_t *handle) H5MM_xfree(handle->secret_id); H5MM_xfree(handle->region); H5MM_xfree(handle->signing_key); + H5MM_xfree(handle->token); HDassert(handle->httpverb != NULL); H5MM_xfree(handle->httpverb); @@ -1163,6 +1165,7 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const H5MM_xfree(handle->region); H5MM_xfree(handle->secret_id); H5MM_xfree(handle->signing_key); + H5MM_xfree(handle->token); if (handle->httpverb != NULL) H5MM_xfree(handle->httpverb); H5MM_xfree(handle); @@ -1226,8 +1229,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) hrb_t *request = NULL; int ret = 0; /* working variable to check */ /* return value of HDsnprintf */ - struct s3r_datastruct *sds = NULL; - herr_t ret_value = SUCCEED; + char *authorization = NULL; + char *buffer1 = NULL; + char *signed_headers = NULL; + struct s3r_datastruct *sds = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -1315,7 +1321,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) else { /* authenticate request */ - char authorization[2048 + 1]; + authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1); /* 2048 := approximate max length... * 67 @@ -1326,10 +1332,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) * + 128 * + 1024 */ - char buffer1[2048 + 1]; /* -> Canonical Request -> Signature */ - char buffer2[256 + 1]; /* -> String To Sign -> Credential */ + char buffer2[256 + 1]; /* -> String To Sign -> Credential */ char iso8601now[ISO8601_SIZE]; - char signed_headers[48 + 1024 + 1]; + buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + + 1); /* -> Canonical Request -> Signature */ + signed_headers = (char *)H5MM_malloc(48 + H5FD_ROS3_MAX_SECRET_KEY_LEN + 1); /* should be large enough for nominal listing: * "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token" * + '\0', with "range;" and/or "x-amz-security-token" possibly absent @@ -1411,7 +1418,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) /**** COMPUTE AUTHORIZATION ****/ /* buffer1 -> canonical request */ - if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 2048, signed_headers, 48 + 1024, request)) { + if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + signed_headers, 48 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + request)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad canonical request"); } /* buffer2->string-to-sign */ @@ -1427,9 +1436,10 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string"); - ret = HDsnprintf(authorization, 2048, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", - buffer2, signed_headers, buffer1); - if (ret <= 0 || ret >= 2048) + ret = HDsnprintf(authorization, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2, + signed_headers, buffer1); + if (ret <= 0 || ret >= 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string"); /* append authorization header to http request buffer */ @@ -1518,6 +1528,18 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) done: /* clean any malloc'd resources */ + if (authorization != NULL) { + H5MM_xfree(authorization); + authorization = NULL; + } + if (buffer1 != NULL) { + H5MM_xfree(buffer1); + buffer1 = NULL; + } + if (signed_headers != NULL) { + H5MM_xfree(signed_headers); + signed_headers = NULL; + } if (curlheaders != NULL) { curl_slist_free_all(curlheaders); curlheaders = NULL; From 10682044b7a891831f779d4b86d4578a1f0ad62d Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Tue, 6 Jun 2023 14:20:34 +0200 Subject: [PATCH 3/8] ROS3: (feature+fix) Temporary security credentials - comments 2 - Implemented the structure H5FD_ros3_fapl_ext_t which extends the existing H5FD_ros3_fapl_ext_t structure. - Adapted h5tools implementation such it makes use of the new introduced H5FD_ros3_fapl_ext_t structure. - Adapted the tool h5dump, h5ls and h5stat to make use of the new introduced structure H5FD_ros3_fapl_ext_t. --- src/H5FDros3.h | 25 +++++++++++++++++++++++-- tools/lib/h5tools.c | 5 +++-- tools/lib/h5tools.h | 1 - tools/libtest/h5tools_test_utils.c | 6 ++---- tools/src/h5dump/h5dump.c | 20 +++++++++++--------- tools/src/h5ls/h5ls.c | 23 ++++++++++++----------- tools/src/h5stat/h5stat.c | 23 ++++++++++++----------- 7 files changed, 63 insertions(+), 40 deletions(-) diff --git a/src/H5FDros3.h b/src/H5FDros3.h index 767b178fb6e..2a8e9676739 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -121,10 +121,10 @@ * A string which specifies the AWS region of the S3 bucket. * * \var char H5FD_ros3_fapl_t::secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1] - * A string which specified the security ID. + * A string which specifies the security ID. * * \var char H5FD_ros3_fapl_t::secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1] - * A string which specified the security key. + * A string which specifies the security key. * */ typedef struct H5FD_ros3_fapl_t { @@ -135,6 +135,27 @@ typedef struct H5FD_ros3_fapl_t { char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]; } H5FD_ros3_fapl_t; +/** + * \struct H5FD_ros3_fapl_ext_t + * \brief Extended configuration structure for holding the configuration data + * to the #H5FD_ROS3 driver via a File Access Property List. + * + * \details H5FD_ros_fapl_ext_t is a public structure that is used to pass + * configuration data to the #H5FD_ROS3 driver via a File Access + * Property List. + * + * \var H5FD_ros3_fapl_t H5FD_ros3_fapl_ext_t::fa + * Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3(). + * + * \var char H5FD_ros3_fapl_ext_t::token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1] + * A string which specifies the session/security token. + * + */ +typedef struct H5FD_ros3_fapl_ext_t { + H5FD_ros3_fapl_t fa; + char token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; +} H5FD_ros3_fapl_ext_t; + #ifdef __cplusplus extern "C" { #endif diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 2219ad6e467..dd88535aade 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -566,10 +566,11 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info) #ifdef H5_HAVE_ROS3_VFD if (!vfd_info->info) H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid"); - if (H5Pset_fapl_ros3(fapl_id, (const H5FD_ros3_fapl_t *)vfd_info->info) < 0) + if (H5Pset_fapl_ros3(fapl_id, &((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->fa) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed"); - if (vfd_info->token && H5Pset_fapl_ros3_token(fapl_id, vfd_info->token) < 0) + if (H5Pset_fapl_ros3_token(fapl_id, ((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->token) < + 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3_token() failed"); #else H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled"); diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index bb6b7e3b693..753a83be480 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -569,7 +569,6 @@ typedef struct h5tools_vfd_info_t { /* Pointer to information to be passed to the driver for its setup */ const void *info; - const char *token; /* Field specifying either the driver's name or value (ID) */ union { diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c index add84a52107..d1560d43b3c 100644 --- a/tools/libtest/h5tools_test_utils.c +++ b/tools/libtest/h5tools_test_utils.c @@ -1070,8 +1070,8 @@ test_set_configured_fapl(void) hid_t fapl_id = H5I_INVALID_HID; other_fa_t wrong_fa = {0x432, 0xf82, 0x9093}; #ifdef H5_HAVE_ROS3_VFD - H5FD_ros3_fapl_t ros3_anon_fa = {1, FALSE, "", "", ""}; - H5FD_ros3_fapl_t ros3_auth_fa = { + H5FD_ros3_fapl_ext_t ros3_anon_fa = {{1, FALSE, "", "", ""}, ""}; + H5FD_ros3_fapl_t ros3_auth_fa = { 1, /* fapl version */ TRUE, /* authenticate */ "us-east-1", /* aws region */ @@ -1181,7 +1181,6 @@ test_set_configured_fapl(void) }; /* testcases `cases` array */ unsigned int i; - char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = ""; #ifdef H5_HAVE_ROS3_VFD n_cases += 3; @@ -1223,7 +1222,6 @@ test_set_configured_fapl(void) vfd_info.type = VFD_BY_NAME; vfd_info.info = C.conf_fa; vfd_info.u.name = C.vfdname; - vfd_info.token = token; result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); if (C.expected == 0) { JSVERIFY(result, H5I_INVALID_HID, C.message) diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index dc398d8c2fd..adee09469b6 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -32,12 +32,15 @@ static hbool_t get_onion_revision_count = FALSE; #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ -static H5FD_ros3_fapl_t ros3_fa_g = { - 1, /* Structure Version */ - FALSE, /* Authenticate? */ - "", /* AWS Region */ - "", /* Access Key ID */ - "", /* Secret Access Key */ +static H5FD_ros3_fapl_ext_t ros3_fa_g = { + { + 1, /* Structure Version */ + FALSE, /* Authenticate? */ + "", /* AWS Region */ + "", /* Access Key ID */ + "", /* Secret Access Key */ + }, + "", /* Session/security token */ }; static char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ #endif /* H5_HAVE_ROS3_VFD */ @@ -1148,7 +1151,7 @@ parse_command_line(int argc, const char *const *argv) case '$': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa_g, token) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &(ros3_fa_g.fa), ros3_fa_g.token) < 0) { error_msg("failed to parse S3 VFD credential info\n"); usage(h5tools_getprogname()); free_handler(hand, argc); @@ -1157,8 +1160,7 @@ parse_command_line(int argc, const char *const *argv) goto done; } - vfd_info_g.info = &ros3_fa_g; - vfd_info_g.token = token; + vfd_info_g.info = &ros3_fa_g; #else error_msg("Read-Only S3 VFD not enabled.\n"); h5tools_setstatus(EXIT_FAILURE); diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index ee92ab6e884..e05414aaf7f 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -2656,15 +2656,17 @@ main(int argc, char *argv[]) #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ - H5FD_ros3_fapl_t ros3_fa = { - 1, /* Structure Version */ - FALSE, /* Authenticate? */ - "", /* AWS Region */ - "", /* Access Key ID */ - "", /* Secret Access Key */ + H5FD_ros3_fapl_ext_t ros3_fa = { + { + 1, /* Structure Version */ + FALSE, /* Authenticate? */ + "", /* AWS Region */ + "", /* Access Key ID */ + "", /* Secret Access Key */ + }, + "", /* Session/security token */ }; - char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ -#endif /* H5_HAVE_ROS3_VFD */ +#endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS /* "Default" HDFS configuration */ @@ -2848,14 +2850,13 @@ main(int argc, char *argv[]) } start++; - if (h5tools_parse_ros3_fapl_tuple(start, ',', &ros3_fa, token) < 0) { + if (h5tools_parse_ros3_fapl_tuple(start, ',', &(ros3_fa.fa), ros3_fa.token) < 0) { HDfprintf(rawerrorstream, "Error: failed to parse S3 VFD credential info\n\n"); usage(); leave(EXIT_FAILURE); } - vfd_info.info = &ros3_fa; - vfd_info.token = token; + vfd_info.info = &ros3_fa; #else HDfprintf(rawerrorstream, "Error: Read-Only S3 VFD is not enabled\n\n"); usage(); diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 58dbf216ee4..a2bb6c4e9d6 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -119,12 +119,15 @@ static const char *drivername = NULL; #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ -static H5FD_ros3_fapl_t ros3_fa = { - 1, /* Structure Version */ - FALSE, /* Authenticate? */ - "", /* AWS Region */ - "", /* Access Key ID */ - "", /* Secret Access Key */ +static H5FD_ros3_fapl_ext_t ros3_fa = { + { + 1, /* Structure Version */ + FALSE, /* Authenticate? */ + "", /* AWS Region */ + "", /* Access Key ID */ + "", /* Secret Access Key */ + }, + "", /* Session/security token */ }; static char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ #endif /* H5_HAVE_ROS3_VFD */ @@ -964,7 +967,7 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re case 'w': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa, token) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &(ros3_fa.fa), ros3_fa.token) < 0) { error_msg("failed to parse S3 VFD credential info\n"); goto error; } @@ -1685,10 +1688,8 @@ main(int argc, char *argv[]) vfd_info.u.name = drivername; #ifdef H5_HAVE_ROS3_VFD - if (!HDstrcmp(drivername, drivernames[ROS3_VFD_IDX])) { - vfd_info.info = &ros3_fa; - vfd_info.token = token; - } + if (!HDstrcmp(drivername, drivernames[ROS3_VFD_IDX])) + vfd_info.info = &ros3_fa; #endif #ifdef H5_HAVE_LIBHDFS if (!HDstrcmp(drivername, drivernames[HDFS_VFD_IDX])) From f8bd2b02d7010c5f1da8e565c0651f8c5ab649c3 Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Fri, 9 Jun 2023 09:55:47 +0200 Subject: [PATCH 4/8] ROS3: (feature+fix) Temporary security credentials - comments 3 - Improved the implementation of adding the session/security to the property list. Now the pointer to the string pointer is stored in the list. Furthermore, also pointer to the string and string itself are managed by the HDF5 library. - Moved the structure H5FD_ros3_fapl_ext_t to the header file h5tools_utils.c. - Adapted the function h5tools_parse_ros3_fapl_tulpe() and h5tools_populate_ros3_fapl() to make use of the structure H5FD_ros3_fapl_ext_t. Adjusted the related tests and the tool h5dump, h5ls and h5stat accordingly. - Implemented additional error checks to ensure that the variable authorization, buffer1 and signed_headers are properly allocated. - Reduced the size of the tmpstr array by 1 in the function H5FD_s3comms_aws_canonical_request(). - Resolved various compiler warnings of h5tools_test_utils tests. - Improved the inline doxygen documentation. --- src/H5FDros3.c | 59 +++-- src/H5FDros3.h | 30 +-- src/H5FDs3comms.c | 13 +- tools/lib/h5tools_utils.c | 31 ++- tools/lib/h5tools_utils.h | 12 +- tools/libtest/h5tools_test_utils.c | 348 ++++++++++++++--------------- tools/src/h5dump/h5dump.c | 2 +- tools/src/h5ls/h5ls.c | 2 +- tools/src/h5stat/h5stat.c | 2 +- 9 files changed, 244 insertions(+), 255 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index b365acba86b..273ad773a99 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -242,8 +242,8 @@ static herr_t H5FD__ros3_validate_config(const H5FD_ros3_fapl_t *fa); static herr_t H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value); static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size); -static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *value); -static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *_value); +static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *_value); static const H5FD_class_t H5FD_ros3_g = { H5FD_CLASS_VERSION, /* struct version */ @@ -626,8 +626,8 @@ H5FD__ros3_fapl_free(void *_fa) herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) { - H5P_genplist_t *plist = NULL; - char token_src[H5FD_ROS3_MAX_SECRET_TOK_LEN] = {0}; + H5P_genplist_t *plist = NULL; + char *token_src; htri_t token_exists; size_t tokenlen = 0; herr_t ret_value = SUCCEED; @@ -651,7 +651,7 @@ H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") if (token_exists) { - if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, token_src) < 0) + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") } @@ -682,7 +682,7 @@ H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) static herr_t H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value) { - char **value = (char **)&_value; + char **value = (char **)_value; herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -716,20 +716,20 @@ H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value) static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size) { - char const *value1 = (char const *)_value1; - char const *value2 = (char const *)_value2; - int ret_value = SUCCEED; + char *const *value1 = (char *const *)_value1; + char *const *value2 = (char *const *)_value2; + int ret_value = SUCCEED; FUNC_ENTER_PACKAGE_NOERR - if (value1) { - if (value2) - ret_value = HDstrcmp(value1, value2); + if (*value1) { + if (*value2) + ret_value = HDstrcmp(*value1, *value2); else ret_value = 1; } else { - if (value2) + if (*value2) ret_value = -1; else ret_value = 0; @@ -753,12 +753,16 @@ H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size) *------------------------------------------------------------------------- */ static herr_t -H5FD__ros3_str_token_close(const char *name, size_t size, void *value) +H5FD__ros3_str_token_close(const char *name, size_t size, void *_value) { + char **value = (char **)_value; herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE_NOERR + if (*value) + HDfree(*value); + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD__ros3_str_token_close */ @@ -778,12 +782,16 @@ H5FD__ros3_str_token_close(const char *name, size_t size, void *value) *------------------------------------------------------------------------- */ static herr_t -H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *value) +H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *_value) { + char **value = (char **)_value; herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE_NOERR + if (*value) + HDfree(*value); + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD__ros3_str_token_delete */ @@ -805,6 +813,7 @@ herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) { H5P_genplist_t *plist = NULL; + char *token_src; htri_t token_exists; herr_t ret_value = SUCCEED; @@ -829,13 +838,17 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") if (token_exists) { - if (H5P_set(plist, ROS3_TOKEN_PROP_NAME, (void *)token) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") + + HDmemcpy(token_src, token, HDstrlen(token) + 1); } else { - if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1), - (void *)token, NULL, NULL, NULL, NULL, H5FD__ros3_str_token_delete, - H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, H5FD__ros3_str_token_close) < 0) + token_src = HDmalloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1)); + HDmemcpy(token_src, token, HDstrlen(token) + 1); + if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char *), &token_src, NULL, NULL, NULL, NULL, + H5FD__ros3_str_token_delete, H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, + H5FD__ros3_str_token_close) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist") } @@ -939,8 +952,8 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) H5FD_ros3_fapl_t fa; H5P_genplist_t *plist = NULL; htri_t token_exists; - char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = {0}; - H5FD_t *ret_value = NULL; + char *token; + H5FD_t *ret_value = NULL; FUNC_ENTER_PACKAGE @@ -973,7 +986,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "failed to check if property token exists in plist") if (token_exists) { - if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, token) < 0) + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get token value") } diff --git a/src/H5FDros3.h b/src/H5FDros3.h index 2a8e9676739..0f2f31538a0 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -135,27 +135,6 @@ typedef struct H5FD_ros3_fapl_t { char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]; } H5FD_ros3_fapl_t; -/** - * \struct H5FD_ros3_fapl_ext_t - * \brief Extended configuration structure for holding the configuration data - * to the #H5FD_ROS3 driver via a File Access Property List. - * - * \details H5FD_ros_fapl_ext_t is a public structure that is used to pass - * configuration data to the #H5FD_ROS3 driver via a File Access - * Property List. - * - * \var H5FD_ros3_fapl_t H5FD_ros3_fapl_ext_t::fa - * Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3(). - * - * \var char H5FD_ros3_fapl_ext_t::token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1] - * A string which specifies the session/security token. - * - */ -typedef struct H5FD_ros3_fapl_ext_t { - H5FD_ros3_fapl_t fa; - char token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; -} H5FD_ros3_fapl_ext_t; - #ifdef __cplusplus extern "C" { #endif @@ -213,10 +192,13 @@ H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token); * \param[in] token Session/security token. * \returns \herr_t * - * \details H5Pset_fapy_ros3_token() modifies the File Access Property List to use the - * #H5FD_ROS3 driver by adding or updating the session/security token. + * \details H5Pset_fapl_ros3_token() modifies an existing File Access Property List which + * is used by #H5FD_ROS3 driver by adding or updating the session/security token + * of the property list. Be aware, to set the token first you need to create + * a proper File Access Property List using H5Pset_fapl_ros() and use this list + * as input argument of the function H5Pset_fapl_ros3_token(). * - * The session token is only needed when you want to access a S3 bucket + * Note, the session token is only needed when you want to access a S3 bucket * using temporary security credentials. * * \since 1.14.2 diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 5ee13f01324..6bec8a317f7 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -1322,6 +1322,8 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) /* authenticate request */ authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1); + if (authorization == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for authorization variable."); /* 2048 := approximate max length... * 67 @@ -1334,9 +1336,13 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) */ char buffer2[256 + 1]; /* -> String To Sign -> Credential */ char iso8601now[ISO8601_SIZE]; - buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + - 1); /* -> Canonical Request -> Signature */ + buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + + 1); /* -> Canonical Request -> Signature */ + if (buffer1 == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for buffer1 variable."); signed_headers = (char *)H5MM_malloc(48 + H5FD_ROS3_MAX_SECRET_KEY_LEN + 1); + if (signed_headers == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for signed_headers variable."); /* should be large enough for nominal listing: * "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token" * + '\0', with "range;" and/or "x-amz-security-token" possibly absent @@ -1664,8 +1670,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c size_t sh_size = (size_t)_sh_size; size_t cr_len = 0; /* working length of canonical request str */ size_t sh_len = 0; /* working length of signed headers str */ - char tmpstr[1024 + 1]; - tmpstr[1024] = 0; /* terminating NULL */ + char tmpstr[1024]; /* "query params" refers to the optional element in the URL, e.g. * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index 3da54973749..178f5175742 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -1028,8 +1028,7 @@ h5tools_getenv_update_hyperslab_bufsize(void) *---------------------------------------------------------------------------- */ herr_t -h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t *fapl_config_out, - char *token_out) +h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_ext_t *fapl_config_out) { const char *ccred[4]; unsigned nelems = 0; @@ -1055,7 +1054,7 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t ccred[3] = (const char *)s3cred[3]; } - if (0 == h5tools_populate_ros3_fapl(fapl_config_out, token_out, ccred)) + if (0 == h5tools_populate_ros3_fapl(fapl_config_out, ccred)) H5TOOLS_GOTO_ERROR(FAIL, "failed to populate S3 VFD FAPL config"); done: @@ -1131,7 +1130,7 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t *---------------------------------------------------------------------------- */ int -h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **values) +h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values) { int show_progress = 0; /* set to 1 for debugging */ int ret_value = 1; /* 1 for success, 0 for failure */ @@ -1149,7 +1148,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **value goto done; } - if (token == NULL) { + if (fa->token == NULL) { if (show_progress) { HDprintf(" ERROR: null pointer to token\n"); } @@ -1160,12 +1159,12 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **value if (show_progress) { HDprintf(" preset fapl with default values\n"); } - fa->version = H5FD_CURR_ROS3_FAPL_T_VERSION; - fa->authenticate = FALSE; - *(fa->aws_region) = '\0'; - *(fa->secret_id) = '\0'; - *(fa->secret_key) = '\0'; - *token = '\0'; + fa->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION; + fa->fa.authenticate = FALSE; + *(fa->fa.aws_region) = '\0'; + *(fa->fa.secret_id) = '\0'; + *(fa->fa.secret_key) = '\0'; + *(fa->token) = '\0'; /* sanity-check supplied values */ @@ -1210,7 +1209,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **value ret_value = 0; goto done; } - HDmemcpy(fa->aws_region, values[0], (HDstrlen(values[0]) + 1)); + HDmemcpy(fa->fa.aws_region, values[0], (HDstrlen(values[0]) + 1)); if (show_progress) { HDprintf(" aws_region set\n"); } @@ -1222,7 +1221,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **value ret_value = 0; goto done; } - HDmemcpy(fa->secret_id, values[1], (HDstrlen(values[1]) + 1)); + HDmemcpy(fa->fa.secret_id, values[1], (HDstrlen(values[1]) + 1)); if (show_progress) { HDprintf(" secret_id set\n"); } @@ -1234,7 +1233,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **value ret_value = 0; goto done; } - HDmemcpy(fa->secret_key, values[2], (HDstrlen(values[2]) + 1)); + HDmemcpy(fa->fa.secret_key, values[2], (HDstrlen(values[2]) + 1)); if (show_progress) { HDprintf(" secret_key set\n"); } @@ -1246,12 +1245,12 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **value ret_value = 0; goto done; } - HDmemcpy(token, values[3], (HDstrlen(values[3]) + 1)); + HDmemcpy(fa->token, values[3], (HDstrlen(values[3]) + 1)); if (show_progress) { HDprintf(" token set\n"); } - fa->authenticate = TRUE; + fa->fa.authenticate = TRUE; if (show_progress) { HDprintf(" set to authenticate\n"); } diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index f1c0e3328c5..e2aceb070d7 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -64,6 +64,14 @@ typedef struct find_objs_t { table_t *dset_table; } find_objs_t; +#ifdef H5_HAVE_ROS3_VFD +/*extended configuration struct for holding the configuration data to the #H5FD_ROS3 driver */ +typedef struct H5FD_ros3_fapl_ext_t { + H5FD_ros3_fapl_t fa; /* ROS3 configuration struct*/ + char token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; /* Session/security token*/ +} H5FD_ros3_fapl_ext_t; +#endif /* H5_HAVE_ROS3_VFD */ + H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for outputting */ /* Definitions of useful routines */ @@ -129,8 +137,8 @@ H5TOOLS_DLL void h5tools_setstatus(int d_status); H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void); #ifdef H5_HAVE_ROS3_VFD H5TOOLS_DLL herr_t h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, - H5FD_ros3_fapl_t *fapl_config_out, char *token); -H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, char *token, const char **values); + H5FD_ros3_fapl_ext_t *fapl_config_out); +H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values); #endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS H5TOOLS_DLL herr_t h5tools_parse_hdfs_fapl_tuple(const char *tuple_str, int delim, diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c index d1560d43b3c..e52b2ec1b0a 100644 --- a/tools/libtest/h5tools_test_utils.c +++ b/tools/libtest/h5tools_test_utils.c @@ -544,11 +544,11 @@ test_parse_tuple(void) success = parse_tuple(tc.in_str, tc.sep, &cpy, &count, &parsed); JSVERIFY(tc.exp_ret, success, "function returned incorrect value") - JSVERIFY(tc.exp_nelems, count, NULL) + JSVERIFY(tc.exp_nelems, count, (char *)NULL) if (success == SUCCEED) { FAIL_IF(parsed == NULL) for (elem_i = 0; elem_i < count; elem_i++) { - JSVERIFY_STR(tc.exp_elems[elem_i], parsed[elem_i], NULL) + JSVERIFY_STR(tc.exp_elems[elem_i], parsed[elem_i], (char *)NULL) } /* TEARDOWN */ HDassert(parsed != NULL); @@ -632,141 +632,133 @@ test_populate_ros3_fa(void) /* NULL fapl config pointer fails */ { - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "a"; - const char *values[] = {"x", "y", "z"}; + const char *values[] = {"x", "y", "z"}; if (show_progress) { HDprintf("NULL fapl pointer\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(NULL, token, values), "fapl pointer cannot be null") + JSVERIFY(0, h5tools_populate_ros3_fapl(NULL, values), "fapl pointer cannot be null") } /* NULL token pointer fails */ { - H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; - const char *values[] = {"x", "y", "z"}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, 0}; + const char *values[] = {"x", "y", "z"}; if (show_progress) { HDprintf("NULL token pointer\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, NULL, values), "token pointer cannot be null") + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "token pointer cannot be null") } /* NULL values pointer yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "x"; + H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, "x"}; if (show_progress) { HDprintf("NULL values pointer\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, NULL), - "NULL values pointer yields \"default\" fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, NULL), "NULL values pointer yields \"default\" fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* all-empty values * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "x"; - const char *values[] = {"", "", "", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, "x"}; + const char *values[] = {"", "", "", ""}; if (show_progress) { HDprintf("all empty values\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "empty values yields \"default\" fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "empty values yields \"default\" fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* successfully set fapl with values * excess value is ignored */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", "y", "z", "a", "b"}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "z", "a", "b"}; if (show_progress) { HDprintf("successful full set\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "four values") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(TRUE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("y", fa.secret_id, NULL) - JSVERIFY_STR("z", fa.secret_key, NULL) - JSVERIFY_STR("a", token, NULL) + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "four values") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(TRUE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("z", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("a", fa.token, (char *)NULL) } /* NULL region * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {NULL, "y", "z", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {NULL, "y", "z", ""}; if (show_progress) { HDprintf("NULL region\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty region * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"", "y", "z", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"", "y", "z", ""}; if (show_progress) { HDprintf("empty region; non-empty id, key\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* region overflow * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"somewhere over the rainbow not too high " - "there is another rainbow bounding some darkened sky", + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"somewhere over the rainbow not too high " + "there is another rainbow bounding some darkened sky", "y", "z", ""}; if (show_progress) { @@ -775,74 +767,71 @@ test_populate_ros3_fa(void) HDassert(HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN); - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* NULL id * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", NULL, "z", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", NULL, "z", ""}; if (show_progress) { HDprintf("NULL id\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty id (non-empty region, key) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", "", "z", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "", "z", ""}; if (show_progress) { HDprintf("empty id; non-empty region and key\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* id overflow * partial set: region */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "Why is it necessary to solve the problem? " - "What benefits will you receive by solving the problem? " - "What is the unknown? " - "What is it you don't yet understand? " - "What is the information you have? " - "What isn't the problem? " - "Is the information insufficient, redundant, or contradictory? " - "Should you draw a diagram or figure of the problem? " - "What are the boundaries of the problem? " - "Can you separate the various parts of the problem?", + "What benefits will you receive by solving the problem? " + "What is the unknown? " + "What is it you don't yet understand? " + "What is the information you have? " + "What isn't the problem? " + "Is the information insufficient, redundant, or contradictory? " + "Should you draw a diagram or figure of the problem? " + "What are the boundaries of the problem? " + "Can you separate the various parts of the problem?", "z", ""}; if (show_progress) { @@ -851,137 +840,131 @@ test_populate_ros3_fa(void) HDassert(HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN); - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* NULL key * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", "y", NULL, ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", NULL, ""}; if (show_progress) { HDprintf("NULL key\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* NULL token * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", "y", "z", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "z", NULL}; if (show_progress) { HDprintf("NULL key\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill token") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill token") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty key (non-empty region, id) * yields authenticating fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", "y", "", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "", ""}; if (show_progress) { HDprintf("empty key; non-empty region and id\n"); } - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(TRUE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("y", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(TRUE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty key, region (non-empty id) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"", "y", "", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"", "y", "", ""}; if (show_progress) { HDprintf("empty key and region; non-empty id\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty key, id (non-empty region) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", "", "", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "", "", ""}; if (show_progress) { HDprintf("empty key and id; non-empty region\n"); } - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* key overflow * partial set: region, id */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = "d"; - const char *values[] = {"x", "y", + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "Why is it necessary to solve the problem? " - "What benefits will you receive by solving the problem? " - "What is the unknown? " - "What is it you don't yet understand? " - "What is the information you have? " - "What isn't the problem? " - "Is the information insufficient, redundant, or contradictory? " - "Should you draw a diagram or figure of the problem? " - "What are the boundaries of the problem? " - "Can you separate the various parts of the problem?", + "What benefits will you receive by solving the problem? " + "What is the unknown? " + "What is it you don't yet understand? " + "What is the information you have? " + "What isn't the problem? " + "Is the information insufficient, redundant, or contradictory? " + "Should you draw a diagram or figure of the problem? " + "What are the boundaries of the problem? " + "Can you separate the various parts of the problem?", ""}; if (show_progress) { @@ -990,25 +973,24 @@ test_populate_ros3_fa(void) HDassert(HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN); - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, token, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("y", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) - JSVERIFY_STR("", token, NULL) + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* use case */ { - H5FD_ros3_fapl_t fa = {0, 0, "", "", ""}; - const char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = ""; - const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA", + H5FD_ros3_fapl_ext_t fa = {{0, 0, "", "", ""}, ""}; + const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA", "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+", ""}; - JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, token, values), "unable to set use case") - JSVERIFY(1, fa.version, "version check") - JSVERIFY(1, fa.authenticate, "should authenticate") + JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "unable to set use case") + JSVERIFY(1, fa.fa.version, "version check") + JSVERIFY(1, fa.fa.authenticate, "should authenticate") } PASSED(); diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index adee09469b6..b6c2f7f66da 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -1151,7 +1151,7 @@ parse_command_line(int argc, const char *const *argv) case '$': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &(ros3_fa_g.fa), ros3_fa_g.token) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa_g) < 0) { error_msg("failed to parse S3 VFD credential info\n"); usage(h5tools_getprogname()); free_handler(hand, argc); diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index e05414aaf7f..0305566f813 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -2850,7 +2850,7 @@ main(int argc, char *argv[]) } start++; - if (h5tools_parse_ros3_fapl_tuple(start, ',', &(ros3_fa.fa), ros3_fa.token) < 0) { + if (h5tools_parse_ros3_fapl_tuple(start, ',', &ros3_fa) < 0) { HDfprintf(rawerrorstream, "Error: failed to parse S3 VFD credential info\n\n"); usage(); leave(EXIT_FAILURE); diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index a2bb6c4e9d6..275b01c2bc1 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -967,7 +967,7 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re case 'w': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &(ros3_fa.fa), ros3_fa.token) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa) < 0) { error_msg("failed to parse S3 VFD credential info\n"); goto error; } From a34438c5ec5fec1f416ae422e6a535e1e7fef6da Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Tue, 20 Jun 2023 07:09:32 +0200 Subject: [PATCH 5/8] ROS3: (feature+fix) Temporary security credentials - comments 4 - Implemented additional error check to ensure that the variable token_src is properly allocated. - Removed the unused static token variable in h5dump.c - Improved the description in the release_docs/RELEASE.txt document about the newly implemented functionality of supporting temporary security credentials for ROS3 file driver. --- release_docs/RELEASE.txt | 13 +++++++++---- src/H5FDros3.c | 4 +++- tools/src/h5dump/h5dump.c | 3 +-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index c875727a6bb..95e8280fd0b 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -148,6 +148,15 @@ New Features Library: -------- + - Implemented support for temporary security credentials for the Read-Only + S3 (ROS3) file driver. + + When using temporary security credentials, one also needs to specify a + session/security token next to the access key id and secret access key. + This token can be specified by the new API function H5Pset_fapl_ros3_token(). + The API function H5Pget_fapl_ros3_token() can be used to retrieve + the currently set token. + - Added a Subfiling VFD configuration file prefix environment variable The Subfiling VFD now checks for values set in a new environment @@ -168,10 +177,6 @@ New Features order to avoid malloc/memcpy. Currently only used for type conversion with selection I/O. - - Added H5Pset_fapl_ros3_token() and H5Pget_fapl_ros3_token() API functions - to set or get the session/security token. This token is needed when - making use of AWS temporary security credentials. - Parallel Library: ----------------- diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 273ad773a99..3399bdfd987 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -844,7 +844,9 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) HDmemcpy(token_src, token, HDstrlen(token) + 1); } else { - token_src = HDmalloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1)); + token_src = (char *)HDmalloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1)); + if (token_src == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for token_src variable."); HDmemcpy(token_src, token, HDstrlen(token) + 1); if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char *), &token_src, NULL, NULL, NULL, NULL, H5FD__ros3_str_token_delete, H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index b6c2f7f66da..7cf800139cd 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -42,8 +42,7 @@ static H5FD_ros3_fapl_ext_t ros3_fa_g = { }, "", /* Session/security token */ }; -static char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ -#endif /* H5_HAVE_ROS3_VFD */ +#endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS /* "Default" HDFS configuration */ From f20169d11d7f4b2a86e0e16662a07948c61ecb81 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Thu, 13 Jul 2023 18:32:23 -0500 Subject: [PATCH 6/8] Remove 'HD' from calls to these system functions that are no longer defined with 'HD' prefix: HDfree HDmemcpy HDassert HDprintf --- src/H5FDros3.c | 12 ++++++------ src/H5FDs3comms.c | 2 +- tools/lib/h5tools_utils.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index a4f240b3454..008d1ab8bc6 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -600,7 +600,7 @@ H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) herr_t ret_value = SUCCEED; FUNC_ENTER_API(FAIL) - H5TRACE2("e", "i*s", fapl_id, token); + H5TRACE3("e", "izx", fapl_id, size, token_dst); #if ROS3_DEBUG HDfprintf(stdout, "H5Pget_fapl_ros3_token() called.\n"); @@ -728,7 +728,7 @@ H5FD__ros3_str_token_close(const char *name, size_t size, void *_value) FUNC_ENTER_PACKAGE_NOERR if (*value) - HDfree(*value); + free(*value); FUNC_LEAVE_NOAPI(ret_value) } /* H5FD__ros3_str_token_close */ @@ -757,7 +757,7 @@ H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void * FUNC_ENTER_PACKAGE_NOERR if (*value) - HDfree(*value); + free(*value); FUNC_LEAVE_NOAPI(ret_value) } /* H5FD__ros3_str_token_delete */ @@ -808,13 +808,13 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") - HDmemcpy(token_src, token, HDstrlen(token) + 1); + memcpy(token_src, token, HDstrlen(token) + 1); } else { - token_src = (char *)HDmalloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1)); + token_src = (char *)malloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1)); if (token_src == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for token_src variable."); - HDmemcpy(token_src, token, HDstrlen(token) + 1); + memcpy(token_src, token, HDstrlen(token) + 1); if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char *), &token_src, NULL, NULL, NULL, NULL, H5FD__ros3_str_token_delete, H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, H5FD__ros3_str_token_close) < 0) diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index a0752737ab1..10b3ff2e48f 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -1382,7 +1382,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header") if (headers == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list."); - HDassert(headers->magic == S3COMMS_HRB_NODE_MAGIC); + assert(headers->magic == S3COMMS_HRB_NODE_MAGIC); } if (rangebytesstr != NULL) { diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index add5379173b..f29b9365484 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -1144,7 +1144,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values) if (fa->token == NULL) { if (show_progress) { - HDprintf(" ERROR: null pointer to token\n"); + printf(" ERROR: null pointer to token\n"); } ret_value = 0; goto done; @@ -1186,7 +1186,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values) } if (values[3] == NULL) { if (show_progress) { - HDprintf(" ERROR: token value cannot be NULL\n"); + printf(" ERROR: token value cannot be NULL\n"); } ret_value = 0; goto done; @@ -1234,14 +1234,14 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values) if (HDstrlen(values[3]) > H5FD_ROS3_MAX_SECRET_TOK_LEN) { if (show_progress) { - HDprintf(" ERROR: token value too long\n"); + printf(" ERROR: token value too long\n"); } ret_value = 0; goto done; } - HDmemcpy(fa->token, values[3], (HDstrlen(values[3]) + 1)); + memcpy(fa->token, values[3], (HDstrlen(values[3]) + 1)); if (show_progress) { - HDprintf(" token set\n"); + printf(" token set\n"); } fa->fa.authenticate = TRUE; From 7bc3d8722fc50af62c7347651bf08d9cc3071359 Mon Sep 17 00:00:00 2001 From: Jordan Henderson Date: Fri, 14 Jul 2023 12:44:25 -0500 Subject: [PATCH 7/8] Fix testing failures and cleanup warnings --- src/H5FDros3.c | 9 ++-- tools/lib/h5tools_utils.c | 8 ---- tools/libtest/h5tools_test_utils.c | 66 ++++++------------------------ tools/src/h5stat/h5stat.c | 1 - 4 files changed, 17 insertions(+), 67 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 008d1ab8bc6..be35abc7aa1 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -647,7 +647,7 @@ H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) *------------------------------------------------------------------------- */ static herr_t -H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value) +H5FD__ros3_str_token_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value) { char **value = (char **)_value; herr_t ret_value = SUCCEED; @@ -681,7 +681,7 @@ H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value) *------------------------------------------------------------------------- */ static int -H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size) +H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t H5_ATTR_UNUSED size) { char *const *value1 = (char *const *)_value1; char *const *value2 = (char *const *)_value2; @@ -720,7 +720,7 @@ H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size) *------------------------------------------------------------------------- */ static herr_t -H5FD__ros3_str_token_close(const char *name, size_t size, void *_value) +H5FD__ros3_str_token_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value) { char **value = (char **)_value; herr_t ret_value = SUCCEED; @@ -749,7 +749,8 @@ H5FD__ros3_str_token_close(const char *name, size_t size, void *_value) *------------------------------------------------------------------------- */ static herr_t -H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *_value) +H5FD__ros3_str_token_delete(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *_value) { char **value = (char **)_value; herr_t ret_value = SUCCEED; diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index f29b9365484..fc9a6462d69 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -1142,14 +1142,6 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values) goto done; } - if (fa->token == NULL) { - if (show_progress) { - printf(" ERROR: null pointer to token\n"); - } - ret_value = 0; - goto done; - } - if (show_progress) { printf(" preset fapl with default values\n"); } diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c index 9969bd03925..87c2ce9b26f 100644 --- a/tools/libtest/h5tools_test_utils.c +++ b/tools/libtest/h5tools_test_utils.c @@ -22,10 +22,6 @@ #define UTIL_TEST_DEBUG 0 -#ifndef __js_test__ - -#define __js_test__ 1L - /***************************************************************************** * * FILE-LOCAL TESTING MACROS @@ -59,7 +55,6 @@ * * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order * FAIL_IF() - check condition - * FAIL_UNLESS() - check _not_ condition * JSVERIFY() - long-int equality check; prints reason/comparison * JSVERIFY_NOT() - long-int inequality check; prints * JSVERIFY_STR() - string equality check; prints @@ -123,31 +118,6 @@ H5_GCC_CLANG_DIAG_OFF("format") goto error; \ } -/*---------------------------------------------------------------------------- - * - * Macro: FAIL_UNLESS() - * - * Purpose: - * - * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", - * e.g., "a != b". - * - * Opposite of FAIL_IF; fails if the given condition is _not_ true. - * - * `FAIL_IF( 5 != my_op() )` - * is equivalent to - * `FAIL_UNLESS( 5 == my_op() )` - * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. - * (see JSVERIFY) - * - *---------------------------------------------------------------------------- - */ -#define FAIL_UNLESS(condition) \ - if (!(condition)) { \ - JSFAILED_AT() \ - goto error; \ - } - /*---------------------------------------------------------------------------- * * Macro: JSERR_LONG() @@ -314,8 +284,6 @@ H5_GCC_CLANG_DIAG_OFF("format") #endif /* ifdef/else JSVERIFY_EXP_ACT */ -#endif /* __js_test__ */ - /* if > 0, be very verbose when performing tests */ #define H5TOOLS_UTILS_TEST_DEBUG 0 @@ -602,19 +570,6 @@ test_populate_ros3_fa(void) JSVERIFY(0, h5tools_populate_ros3_fapl(NULL, values), "fapl pointer cannot be null") } - /* NULL token pointer fails - */ - { - H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, 0}; - const char *values[] = {"x", "y", "z"}; - - if (show_progress) { - HDprintf("NULL token pointer\n"); - } - - JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "token pointer cannot be null") - } - /* NULL values pointer yields default fapl */ { @@ -838,7 +793,7 @@ test_populate_ros3_fa(void) const char *values[] = {"x", "y", "z", NULL}; if (show_progress) { - HDprintf("NULL key\n"); + printf("NULL key\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill token") @@ -1011,13 +966,6 @@ test_set_configured_fapl(void) other_fa_t wrong_fa = {0x432, 0xf82, 0x9093}; #ifdef H5_HAVE_ROS3_VFD H5FD_ros3_fapl_ext_t ros3_anon_fa = {{1, FALSE, "", "", ""}, ""}; - H5FD_ros3_fapl_t ros3_auth_fa = { - 1, /* fapl version */ - TRUE, /* authenticate */ - "us-east-1", /* aws region */ - "12345677890abcdef", /* simulate access key ID */ - "oiwnerwe9u0234nJw0-aoj+dsf", /* simulate secret key */ - }; #endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS H5FD_hdfs_fapl_t hdfs_fa = { @@ -1162,7 +1110,17 @@ test_set_configured_fapl(void) vfd_info.type = VFD_BY_NAME; vfd_info.info = C.conf_fa; vfd_info.u.name = C.vfdname; - result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + + if (C.expected == 1) + result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + else { + H5E_BEGIN_TRY + { + result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + } + H5E_END_TRY; + } + if (C.expected == 0) { JSVERIFY(result, H5I_INVALID_HID, C.message) } diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index b5b0bbaf418..46984f11a45 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -129,7 +129,6 @@ static H5FD_ros3_fapl_ext_t ros3_fa = { }, "", /* Session/security token */ }; -static char token[H5FD_ROS3_MAX_SECRET_TOK_LEN]; /* Session/security token */ #endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS From d03d91a91622c1e683a577ea9df1cac477d2f339 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Jul 2023 17:45:51 +0000 Subject: [PATCH 8/8] Committing clang-format changes --- tools/src/h5stat/h5stat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 46984f11a45..5cfa61d182f 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -129,7 +129,7 @@ static H5FD_ros3_fapl_ext_t ros3_fa = { }, "", /* Session/security token */ }; -#endif /* H5_HAVE_ROS3_VFD */ +#endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS /* "Default" HDFS configuration */