Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ROS3: (feature) Temporary security credentials #3030

Merged
merged 9 commits into from
Jul 14, 2023
Merged

ROS3: (feature) Temporary security credentials #3030

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
4d4bb1d
ROS3: (feature) Temporary security credentials
jwsblokland May 15, 2023
f263a4a
ROS3: (feature+fix) Temporary security credentials - comments
jwsblokland Jun 5, 2023
1068204
ROS3: (feature+fix) Temporary security credentials - comments 2
jwsblokland Jun 6, 2023
f8bd2b0
ROS3: (feature+fix) Temporary security credentials - comments 3
jwsblokland Jun 9, 2023
a34438c
ROS3: (feature+fix) Temporary security credentials - comments 4
jwsblokland Jun 20, 2023
e6233bf
Merge branch 'develop' into feature/ros3-temporary_credentials
lrknox Jul 7, 2023
f20169d
Remove 'HD' from calls to these system functions that are no longer
lrknox Jul 13, 2023
7bc3d87
Fix testing failures and cleanup warnings
jhendersonHDF Jul 14, 2023
d03d91a
Committing clang-format changes
github-actions[bot] Jul 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions release_docs/RELEASE.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,10 @@ New Features
order to avoid malloc/memcpy. Currently only used for type conversion
with selection I/O.

- Added H5Pset_fapl_ros3_token() and H5Pget_fapl_ros3_token() API functions
to set or get the session/security token. This token is needed when
making use of AWS temporary security credentials.


Parallel Library:
-----------------
Expand Down
268 changes: 264 additions & 4 deletions src/H5FDros3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@
*/
static hid_t H5FD_ROS3_g = 0;

/* Session/security token property name
*/
#define ROS3_TOKEN_PROP_NAME "ros3_token_prop"

#if ROS3_STATS

/* arbitrarily large value, such that any reasonable size read will be "less"
Expand Down Expand Up @@ -236,6 +240,11 @@ static herr_t H5FD__ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing

static herr_t H5FD__ros3_validate_config(const H5FD_ros3_fapl_t *fa);

static herr_t H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value);
static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size);
static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *value);
static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *value);

static const H5FD_class_t H5FD_ros3_g = {
H5FD_CLASS_VERSION, /* struct version */
H5FD_ROS3_VALUE, /* value */
Expand Down Expand Up @@ -599,6 +608,240 @@ H5FD__ros3_fapl_free(void *_fa)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD__ros3_fapl_free() */

/*-------------------------------------------------------------------------
* Function: H5Pget_fapl_ros3_token
*
* Purpose: Returns session/secutiry token of the ros3 file access
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
* property list though the function arguments.
*
* Return: Success: Non-negative
*
* Failure: Negative
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/)
{
H5P_genplist_t *plist = NULL;
char token_src[H5FD_ROS3_MAX_SECRET_TOK_LEN] = {0};
htri_t token_exists;
size_t tokenlen = 0;
herr_t ret_value = SUCCEED;

FUNC_ENTER_API(FAIL)
H5TRACE2("e", "i*s", fapl_id, token);

#if ROS3_DEBUG
HDfprintf(stdout, "H5Pget_fapl_ros3_token() called.\n");
#endif

if (size == 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size cannot be zero.")
if (token_dst == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "token_dst is NULL")

if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list")
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist")
if (token_exists) {
if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, token_src) < 0)
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value")
}

/* Copy the token data out */
tokenlen = HDstrlen(token_src);
if (size < tokenlen) {
tokenlen = size - 1;
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
}
H5MM_memcpy(token_dst, token_src, sizeof(char) * tokenlen);
token_dst[tokenlen] = '\0';
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved

done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_fapl_ros3_token() */

/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_copy()
*
* Purpose: Create a copy of the token string.
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value)
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
{
char **value = (char **)&_value;
herr_t ret_value = SUCCEED;

FUNC_ENTER_PACKAGE

#if ROS3_DEBUG
HDfprintf(stdout, "H5FD__ros3_str_token_copy() called.\n");
#endif

if (*value)
if (NULL == (*value = HDstrdup(*value)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy string property token")

done:
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD__ros3_str_token_copy() */

/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_cmp()
*
* Purpose: Compares two token strings with each other.
*
* Return:
* - Equivalent: 0
* - Not Equivalent: non-zero value
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static int
H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size)
{
char const *value1 = (char const *)_value1;
char const *value2 = (char const *)_value2;
int ret_value = SUCCEED;

FUNC_ENTER_PACKAGE_NOERR

if (value1) {
if (value2)
ret_value = HDstrcmp(value1, value2);
else
ret_value = 1;
}
else {
if (value2)
ret_value = -1;
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
else
ret_value = 0;
}

FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_token_cmp */

/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_close()
*
* Purpose: Closes/frees the memory associated to the token string.
* Currently, it is an empty implementation since there no
* additional treatment needed for this property.
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_token_close(const char *name, size_t size, void *value)
{
herr_t ret_value = SUCCEED;

FUNC_ENTER_PACKAGE_NOERR

FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_token_close */

/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_token_delete()
*
* Purpose: Deletes the property token from the property list and frees
* the memory associated to the token string.
* Currently, it is an empty implementation since there no
* additional treatment needed for this property.
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *value)
{
herr_t ret_value = SUCCEED;

FUNC_ENTER_PACKAGE_NOERR
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved

FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_token_delete */

/*-------------------------------------------------------------------------
* Function: H5Pset_fapl_ros3_token()
*
* Purpose: Modify the file access property list to use the H5FD_ROS3
* driver defined in this source file.
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
*
* Return: SUCCEED/FAIL
*
* Programmer: Jan-Willem Blokland
* 2023-05-26
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
{
H5P_genplist_t *plist = NULL;
htri_t token_exists;
herr_t ret_value = SUCCEED;

FUNC_ENTER_API(FAIL)
H5TRACE2("e", "i*s", fapl_id, token);

#if ROS3_DEBUG
HDfprintf(stdout, "H5Pset_fapl_ros3_token() called.\n");
#endif

if (fapl_id == H5P_DEFAULT)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list")
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list")
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
if (HDstrlen(token) >= H5FD_ROS3_MAX_SECRET_TOK_LEN)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL,
"specified token exceeds the internally specified maximum string length")

if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist")

if (token_exists) {
if (H5P_set(plist, ROS3_TOKEN_PROP_NAME, (void *)token) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value")
}
else {
if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char) * H5FD_ROS3_MAX_SECRET_TOK_LEN,
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
(void *)token, NULL, NULL, NULL, NULL, H5FD__ros3_str_token_delete,
H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, H5FD__ros3_str_token_close) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist")
}

done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pset_fapl_ros3_token() */

#if ROS3_STATS
/*----------------------------------------------------------------------------
*
Expand Down Expand Up @@ -693,7 +936,10 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
unsigned char signing_key[SHA256_DIGEST_LENGTH];
s3r_t *handle = NULL;
H5FD_ros3_fapl_t fa;
H5FD_t *ret_value = NULL;
H5P_genplist_t *plist = NULL;
htri_t token_exists;
char token[H5FD_ROS3_MAX_SECRET_TOK_LEN] = {0};
H5FD_t *ret_value = NULL;

FUNC_ENTER_PACKAGE

Expand All @@ -720,6 +966,16 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)")

/* Session/security token */
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, NULL, "not a file access property list")
if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "failed to check if property token exists in plist")
if (token_exists) {
if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, token) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get token value")
}

/* open file; procedure depends on whether or not the fapl instructs to
* authenticate requests or not.
*/
Expand All @@ -736,11 +992,15 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
(const char *)fa.aws_region, (const char *)iso8601now))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key")

handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id,
(const unsigned char *)signing_key);
if (token_exists)
handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id,
(const unsigned char *)signing_key, (const char *)token);
else
handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id,
(const unsigned char *)signing_key, "");
}
else
handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL);
handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL, NULL);

if (handle == NULL)
/* If we want to check CURL's say on the matter in a controlled
Expand Down
22 changes: 22 additions & 0 deletions src/H5FDros3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,21 @@
*
* String: "Secret Access Key" associated with the ID and resource.
*
* `secret_tok` (char[])
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved
*
* String: "Security/Session Token" associated with the ID and resource.
*
* This secret_tok is only needed when you are using temporary security
* credentials.
*
****************************************************************************/

#define H5FD_CURR_ROS3_FAPL_T_VERSION 1

#define H5FD_ROS3_MAX_REGION_LEN 32
#define H5FD_ROS3_MAX_SECRET_ID_LEN 128
#define H5FD_ROS3_MAX_SECRET_KEY_LEN 128
#define H5FD_ROS3_MAX_SECRET_TOK_LEN 1024

typedef struct H5FD_ros3_fapl_t {
int32_t version;
Expand Down Expand Up @@ -104,6 +112,20 @@ H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out);
*/
H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa);

/**
* \ingroup FAPL
*
* \todo Add missing documentation
*/
H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token);
jhendersonHDF marked this conversation as resolved.
Show resolved Hide resolved

/**
* \ingroup FAPL
*
* \todo Add missing documentation
*/
H5_DLL herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token);

#ifdef __cplusplus
}
#endif
Expand Down
Loading