diff --git a/doc/changelog.md b/doc/changelog.md index 9699e681..c9a26d81 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -6,10 +6,20 @@ To be released at a future time. Description +- Add Client API functions to put, get, unpack, + delete, poll, and check for existance of raw bytes for the + C++ and Python clients. +- Fix lint issues using make lint - Reenable move semantics and fix compiler warnings. Detailed Notes +- Add Client API functions to put, get, unpack, + delete, poll, and check for existance of raw bytes for the + C++ and Python clients. + ([PR521](https://github.com/CrayLabs/SmartRedis/pull/521)) +- Fix lint issues using make lint + ([PR522](https://github.com/CrayLabs/SmartRedis/pull/522)) - Fix compiler warnings stemming from override and const keywords as well as move semantics impliclty disabled because of std::random_device. @@ -21,14 +31,11 @@ Released on 27 September, 2024 Description -- Fix lint issues using make lint - Fix RedisAI build to allow for compilation with GCC-14 - Fix a memory leak in the Fortran Dataset implementation Detailed Notes -- Fix lint issues using make lint - ([PR522](https://github.com/CrayLabs/SmartRedis/pull/522)) - Fix RedisAI build to allow for compilation with GCC-14. Also, we only use the Torch backend and change the compilation of RedisAI to use CMake (like SmartSim) diff --git a/examples/serial/cpp/CMakeLists.txt b/examples/serial/cpp/CMakeLists.txt index 89ce0b65..30370408 100644 --- a/examples/serial/cpp/CMakeLists.txt +++ b/examples/serial/cpp/CMakeLists.txt @@ -50,6 +50,7 @@ list(APPEND EXECUTABLES smartredis_dataset smartredis_model smartredis_mnist + smartredis_put_get_bytes ) # Build the examples diff --git a/examples/serial/cpp/smartredis_put_get_bytes.cpp b/examples/serial/cpp/smartredis_put_get_bytes.cpp new file mode 100644 index 00000000..8679120c --- /dev/null +++ b/examples/serial/cpp/smartredis_put_get_bytes.cpp @@ -0,0 +1,69 @@ +/* + * BSD 2-Clause License + * + * Copyright (c) 2021-2024, Hewlett Packard Enterprise + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "client.h" +#include +#include + +int main(int argc, char* argv[]) { + + // Initialize some byte data + size_t n_bytes = 255; + std::vector input_bytes(n_bytes); + for(size_t i = 0; i < n_bytes; i++) { + input_bytes[i] = i; + } + + SmartRedis::Client client("client_test_put_get_bytes"); + + std::string key = "put_get_bytes_test"; + + client.put_bytes(key, input_bytes.data(), n_bytes); + + std::vector output_bytes(n_bytes, 0); + + size_t received_bytes = 0; + client.unpack_bytes(key, output_bytes.data(), n_bytes, + received_bytes); + + if (received_bytes != n_bytes) { + std::cout<<"Output byte size "<put_bytes(key, bytes, n_bytes); + + if (reply.has_error()) + throw SRRuntimeException("put_bytes failed"); +} + +// Get byte data and return to the user via modifying the +// supplied void pointer. +void Client::get_bytes(const std::string& name, + void*& data, + size_t& n_bytes) +{ + std::string get_key = _build_bytes_key(name, true); + CommandReply reply = _redis_server->get_bytes(get_key); + + if (reply.has_error()) + throw SRRuntimeException("put_bytes failed"); + + n_bytes = reply.str_len(); + data = malloc(n_bytes); + std::memcpy(data, (void*)(reply.str()), n_bytes); +} + +// Get byte data and fill an already allocated array +// memory space that has the specified size. +void Client::unpack_bytes(const std::string& name, + void* data, + const size_t n_bytes, + size_t& n_used_bytes) +{ + // Track calls to this API function + LOG_API_FUNCTION(); + + std::string get_key = _build_bytes_key(name, true); + CommandReply reply = _redis_server->get_bytes(get_key); + + if (reply.has_error()) + throw SRRuntimeException("put_bytes failed"); + + if (n_bytes < reply.str_len()) { + throw SRRuntimeException("Provided number of bytes of " + + std::to_string(n_bytes) + " " + + "smaller than retrieved data size of " + + std::to_string(reply.str_len()) + "."); + } + + n_used_bytes = reply.str_len(); + std::memcpy(data, reply.str(), reply.str_len()); +} + +// Delete bytes from the database +void Client::delete_bytes(const std::string& name) +{ + // Track calls to this API function + LOG_API_FUNCTION(); + + std::string key = _build_bytes_key(name, true); + CommandReply reply = _redis_server->delete_bytes(key); + if (reply.has_error()) + throw SRRuntimeException("put_bytes failed"); +} + // Get the tensor data, dimensions, and type for the provided tensor name. // This function will allocate and retain management of the memory for the // tensor data. @@ -1132,6 +1205,17 @@ bool Client::model_exists(const std::string& name) return _redis_server->model_key_exists(key); } +// Check if the bytes exists in the database +bool Client::bytes_exists(const std::string& name) +{ + // Track calls to this API function + LOG_API_FUNCTION(); + + std::string key = _build_bytes_key(name, true); + return _redis_server->key_exists(key); +} + + // Check if the key exists in the database at a specified frequency for a specified number of times bool Client::poll_key(const std::string& key, int poll_frequency_ms, @@ -1208,6 +1292,25 @@ bool Client::poll_dataset(const std::string& name, return false; } +// Check if the bytes exists in the database at a specified frequency for a specified number of times +bool Client::poll_bytes(const std::string& name, + int poll_frequency_ms, + int num_tries) +{ + // Track calls to this API function + LOG_API_FUNCTION(); + + // Check for the tensor however many times requested + for (int i = 0; i < num_tries; i++) { + if (bytes_exists(name)) + return true; + std::this_thread::sleep_for(std::chrono::milliseconds(poll_frequency_ms)); + } + + // If we get here, it was never found + return false; +} + // Establish a datasource void Client::set_data_source(std::string source_id) { @@ -1237,6 +1340,36 @@ void Client::set_data_source(std::string source_id) _get_key_prefix = _get_key_prefixes[save_index]; } +// Set whether names of tensor entities should be prefixed +// (e.g. in an ensemble) to form database keys. Prefixes will only be used +// if they were previously set through the environment variables SSKEYOUT +// and SSKEYIN. Keys of entities created before this function is called +// will not be affected. By default, the client prefixes tensor and dataset +// keys with the first prefix specified with the SSKEYIN and SSKEYOUT +// environment variables. +void Client::use_tensor_ensemble_prefix(bool use_prefix) +{ + // Track calls to this API function + LOG_API_FUNCTION(); + + _use_tensor_prefix = use_prefix; +} + +// Set whether names of dataset entities should be prefixed +// (e.g. in an ensemble) to form database keys. Prefixes will only be used +// if they were previously set through the environment variables SSKEYOUT +// and SSKEYIN. Keys of entities created before this function is called +// will not be affected. By default, the client prefixes dataset +// keys with the first prefix specified with the SSKEYIN and SSKEYOUT +// environment variables. +void Client::use_dataset_ensemble_prefix(bool use_prefix) +{ + // Track calls to this API function + LOG_API_FUNCTION(); + + _use_dataset_prefix = use_prefix; +} + // Set whether names of model and script entities should be prefixed // (e.g. in an ensemble) to form database keys. Prefixes will only be // used if they were previously set through the environment variables @@ -1256,7 +1389,8 @@ void Client::use_model_ensemble_prefix(bool use_prefix) // used if they were previously set through the environment variables // SSKEYOUT and SSKEYIN. Keys of entities created before this function // is called will not be affected. By default, the client prefixes -// aggregation list keys. +// aggregation list keys with the first prefix specified with the +// SSKEYIN and SSKEYOUT environment variables. void Client::use_list_ensemble_prefix(bool use_prefix) { // Track calls to this API function @@ -1265,35 +1399,19 @@ void Client::use_list_ensemble_prefix(bool use_prefix) _use_list_prefix = use_prefix; } - -// Set whether names of tensor entities should be prefixed -// (e.g. in an ensemble) to form database keys. Prefixes will only be used -// if they were previously set through the environment variables SSKEYOUT -// and SSKEYIN. Keys of entities created before this function is called -// will not be affected. By default, the client prefixes tensor and dataset -// keys with the first prefix specified with the SSKEYIN and SSKEYOUT -// environment variables. -void Client::use_tensor_ensemble_prefix(bool use_prefix) -{ - // Track calls to this API function - LOG_API_FUNCTION(); - - _use_tensor_prefix = use_prefix; -} - -// Set whether names of dataset entities should be prefixed -// (e.g. in an ensemble) to form database keys. Prefixes will only be used -// if they were previously set through the environment variables SSKEYOUT -// and SSKEYIN. Keys of entities created before this function is called -// will not be affected. By default, the client prefixes dataset -// keys with the first prefix specified with the SSKEYIN and SSKEYOUT -// environment variables. -void Client::use_dataset_ensemble_prefix(bool use_prefix) +// Set whether names of raw bytes should be prefixed +// (e.g. in an ensemble) to form database keys. Prefixes will only be +// used if they were previously set through the environment variables +// SSKEYOUT and SSKEYIN. Keys of entities created before this function +// is called will not be affected. By default, the client prefixes +// raw byte keys with the first prefix specified with the +// SKEYIN and SSKEYOUT environment variables. +void Client::use_bytes_ensemble_prefix(bool use_prefix) { // Track calls to this API function LOG_API_FUNCTION(); - _use_dataset_prefix = use_prefix; + _use_bytes_prefix = use_prefix; } // Returns information about the given database node @@ -2040,6 +2158,16 @@ Client::_build_list_key(const std::string& list_name, return prefix + list_name; } +// Build full formatted key of a tensor, based on current prefix settings. +inline std::string Client::_build_bytes_key(const std::string& key, + bool on_db) +{ + std::string prefix(""); + if (_use_bytes_prefix) + prefix = on_db ? _get_prefix() : _put_prefix(); + + return prefix + key; +} // Create the key to place an indicator in the database that the // dataset has been successfully stored. diff --git a/src/cpp/redis.cpp b/src/cpp/redis.cpp index f730ed50..33aa3f0e 100644 --- a/src/cpp/redis.cpp +++ b/src/cpp/redis.cpp @@ -185,6 +185,42 @@ CommandReply Redis::put_tensor(TensorBase& tensor) return run(cmd); } +// Put bytes on the server +CommandReply Redis::put_bytes(const std::string& key, + const void* bytes, + const size_t n_bytes) +{ + // Build the command + SingleKeyCommand cmd; + cmd << "SET" << Keyfield(key) + << std::string_view((char*)bytes, n_bytes); + + // Run it + return run(cmd); +} + +// Get bytes from the server +CommandReply Redis::get_bytes(const std::string& key) +{ + // Build the command + SingleKeyCommand cmd; + cmd << "GET" << Keyfield(key); + + // Run it + return run(cmd); +} + +// Delete bytes on the server +CommandReply Redis::delete_bytes(const std::string& key) +{ + // Build the command + SingleKeyCommand cmd; + cmd << "UNLINK" << Keyfield(key); + + // Run it + return run(cmd); +} + // Get a Tensor from the server CommandReply Redis::get_tensor(const std::string& key) { diff --git a/src/cpp/rediscluster.cpp b/src/cpp/rediscluster.cpp index f73c4795..f9ebea4e 100644 --- a/src/cpp/rediscluster.cpp +++ b/src/cpp/rediscluster.cpp @@ -382,6 +382,42 @@ CommandReply RedisCluster::put_tensor(TensorBase& tensor) return run(cmd); } +// Put bytes on the server +CommandReply RedisCluster::put_bytes(const std::string& key, + const void* bytes, + const size_t n_bytes) +{ + // Build the command + SingleKeyCommand cmd; + cmd << "SET" << Keyfield(key) + << std::string_view((char*)bytes, n_bytes); + + // Run it + return run(cmd); +} + +// Get bytes from the server +CommandReply RedisCluster::get_bytes(const std::string& key) +{ + // Build the command + SingleKeyCommand cmd; + cmd << "GET" << Keyfield(key); + + // Run it + return run(cmd); +} + +// Delete bytes on the server +CommandReply RedisCluster::delete_bytes(const std::string& key) +{ + // Build the command + SingleKeyCommand cmd; + cmd << "UNLINK" << Keyfield(key); + + // Run it + return run(cmd); +} + // Get a Tensor from the server CommandReply RedisCluster::get_tensor(const std::string& key) { diff --git a/src/python/bindings/bind.cpp b/src/python/bindings/bind.cpp index 11ff4e9a..f80cd768 100644 --- a/src/python/bindings/bind.cpp +++ b/src/python/bindings/bind.cpp @@ -62,6 +62,9 @@ PYBIND11_MODULE(smartredisPy, m) { .def(py::init()) .def(py::init()) .CLIENT_METHOD(put_tensor) + .CLIENT_METHOD(put_bytes) + .CLIENT_METHOD(get_bytes) + .CLIENT_METHOD(delete_bytes) .CLIENT_METHOD(get_tensor) .CLIENT_METHOD(delete_tensor) .CLIENT_METHOD(copy_tensor) @@ -94,14 +97,17 @@ PYBIND11_MODULE(smartredisPy, m) { .CLIENT_METHOD(model_exists) .CLIENT_METHOD(tensor_exists) .CLIENT_METHOD(dataset_exists) + .CLIENT_METHOD(bytes_exists) .CLIENT_METHOD(poll_model) .CLIENT_METHOD(poll_tensor) .CLIENT_METHOD(poll_dataset) + .CLIENT_METHOD(poll_bytes) .CLIENT_METHOD(set_data_source) .CLIENT_METHOD(use_tensor_ensemble_prefix) .CLIENT_METHOD(use_dataset_ensemble_prefix) .CLIENT_METHOD(use_model_ensemble_prefix) .CLIENT_METHOD(use_list_ensemble_prefix) + .CLIENT_METHOD(use_bytes_ensemble_prefix) .CLIENT_METHOD(get_db_node_info) .CLIENT_METHOD(get_db_cluster_info) .CLIENT_METHOD(get_ai_info) diff --git a/src/python/module/smartredis/client.py b/src/python/module/smartredis/client.py index 36e46d3d..294c8f65 100644 --- a/src/python/module/smartredis/client.py +++ b/src/python/module/smartredis/client.py @@ -26,6 +26,7 @@ # pylint: disable=too-many-lines,too-many-public-methods import inspect +import io import os import os.path as osp import typing as t @@ -188,6 +189,24 @@ def put_tensor(self, name: str, data: np.ndarray) -> None: else: self._client.put_tensor(name, dtype, data.copy()) + @exception_handler + def put_bytes(self, name: str, data: io.BytesIO) -> None: + """Put bytes to a Redis database + + The final key under which the bytes are stored + may be formed by applying a prefix to the supplied + name. See use_bytes_ensemble_prefix() for more details. + + :param name: name for bytes to be stored at + :type name: str + :param data: bytes to be stored + :type data: io.BytesIO + :raises RedisReplyError: if put fails + """ + typecheck(name, "name", str) + typecheck(data, "data", io.BytesIO) + self._client.put_bytes(name, data) + @exception_handler def get_tensor(self, name: str) -> np.ndarray: """Get a tensor from the database @@ -206,6 +225,25 @@ def get_tensor(self, name: str) -> np.ndarray: typecheck(name, "name", str) return self._client.get_tensor(name) + @exception_handler + def get_bytes(self, name: str) -> str: + """Get bytes from the database + + The key used to locate the bytes + may be formed by applying a prefix to the supplied + name. See set_data_source() + and use_bytes_ensemble_prefix() for more details. + + :param name: name associated with the bytes + :type name: str + :raises RedisReplyError: if get fails + :return: data bytes + :rtype: io.BytesIO + """ + + typecheck(name, "name", str) + return io.BytesIO(self._client.get_bytes(name)) + @exception_handler def delete_tensor(self, name: str) -> None: """Delete a tensor from the database @@ -222,6 +260,22 @@ def delete_tensor(self, name: str) -> None: typecheck(name, "name", str) self._client.delete_tensor(name) + @exception_handler + def delete_bytes(self, name: str) -> None: + """Delete a bytes from the database + + The key used to locate the bytes to be deleted + may be formed by applying a prefix to the supplied + name. See set_data_source() + and use_bytes_ensemble_prefix() for more details. + + :param name: name bytes is stored at + :type name: str + :raises RedisReplyError: if deletion fails + """ + typecheck(name, "name", str) + self._client.delete_bytes(name) + @exception_handler def copy_tensor(self, src_name: str, dest_name: str) -> None: """Copy a tensor at one name to another name @@ -1131,6 +1185,24 @@ def model_exists(self, name: str) -> bool: typecheck(name, "name", str) return self._client.model_exists(name) + @exception_handler + def bytes_exists(self, name: str) -> bool: + """Check if bytes exists in the database + + The key used to check for existence + may be formed by applying a prefix to the supplied + name. See set_data_source() + and use_bytes_ensemble_prefix() for more details. + + :param name: The bytes name that will be checked in the database + :type name: str + :returns: Returns true if the bytes exists in the database + :rtype: bool + :raises RedisReplyError: if checking for bytes existence causes an error + """ + typecheck(name, "name", str) + return self._client.bytes_exists(name) + @exception_handler def key_exists(self, key: str) -> bool: """Check if the key exists in the database @@ -1248,6 +1320,33 @@ def poll_model(self, name: str, poll_frequency_ms: int, num_tries: int) -> bool: typecheck(num_tries, "num_tries", int) return self._client.poll_model(name, poll_frequency_ms, num_tries) + @exception_handler + def poll_bytes(self, name: str, poll_frequency_ms: int, num_tries: int) -> bool: + """Check if a bytes exists in the database + + The check is repeated at a specified polling interval and for + a specified number of retries. + The bytes key used to check for existence + may be formed by applying a prefix to the supplied + name. See set_data_source() + and use_bytes_ensemble_prefix() for more details. + + :param name: The bytes name that will be checked in the database + :type name: str + :param poll_frequency_ms: The polling interval, in milliseconds + :type poll_frequency_ms: int + :param num_tries: The total number of retries for the check + :type num_tries: int + :returns: Returns true if the bytes key is found within the + specified number of tries, otherwise false. + :rtype: bool + :raises RedisReplyError: if an error occurs while polling + """ + typecheck(name, "name", str) + typecheck(poll_frequency_ms, "poll_frequency_ms", int) + typecheck(num_tries, "num_tries", int) + return self._client.poll_bytes(name, poll_frequency_ms, num_tries) + @exception_handler def set_data_source(self, source_id: str) -> None: """Set the data source, a key prefix for future operations @@ -1373,6 +1472,28 @@ def use_dataset_ensemble_prefix(self, use_prefix: bool) -> None: typecheck(use_prefix, "use_prefix", bool) return self._client.use_dataset_ensemble_prefix(use_prefix) + @exception_handler + def use_bytes_ensemble_prefix(self, use_prefix: bool) -> None: + """Control whether byte keys are prefixed (e.g. in an + ensemble) when forming database keys + + This function can be used to avoid key collisions in an ensemble + by prepending the string value from the environment variable SSKEYIN + to tensor names. + Prefixes will only be used if they were previously set through + environment variables SSKEYIN and SSKEYOUT. + Keys for entities created before this function is called + will not be retroactively prefixed. + By default, the client prefixes byte keys when a prefix is + available. + + :param use_prefix: If set to true, all future operations on raw bytes + will use a prefix, if available. + :type use_prefix: bool + """ + typecheck(use_prefix, "use_prefix", bool) + return self._client.use_bytes_ensemble_prefix(use_prefix) + @exception_handler def get_db_node_info(self, addresses: t.List[str]) -> t.List[t.Dict]: """Returns information about given database nodes diff --git a/src/python/module/smartredis/error.py b/src/python/module/smartredis/error.py index e00fe392..f93e6a5e 100644 --- a/src/python/module/smartredis/error.py +++ b/src/python/module/smartredis/error.py @@ -73,7 +73,7 @@ def _check_error(cpp_error: str, method: str = "", key: str = "") -> str: if method: msg = f"{method} execution failed\n" if "REDIS_REPLY_NIL" in cpp_error: - msg += f"No Dataset stored at key: {key}" + msg += f"No data stored at key: {key}" return msg msg += cpp_error return msg diff --git a/src/python/src/pyclient.cpp b/src/python/src/pyclient.cpp index f521418b..0461ad78 100644 --- a/src/python/src/pyclient.cpp +++ b/src/python/src/pyclient.cpp @@ -125,6 +125,14 @@ void PyClient::put_tensor( }); } +void PyClient::put_bytes(std::string& name, py::object data) +{ + MAKE_CLIENT_API({ + std::string bytes_data = data.attr("getvalue")().cast(); + _client->put_bytes(name, bytes_data.data(), bytes_data.size()); + }); +} + py::array PyClient::get_tensor(const std::string& name) { return MAKE_CLIENT_API({ @@ -184,6 +192,35 @@ py::array PyClient::get_tensor(const std::string& name) }); } +// Get a py::bytes object pointing to the underlying bytes data +py::bytes PyClient::get_bytes(const std::string& name) +{ + return MAKE_CLIENT_API({ + + void* data = NULL; + size_t n_bytes = 0; + + // Get the bytes and store in data pointer and update n_bytes + _client->get_bytes(name, data, n_bytes); + + // TODO by using py::bytes makes another copy. It would + // be ideal to transfer ownership because get_bytes gives + // back new, unmanaged memory that could be transferred + py::bytes py_bytes = py::bytes((char*)data, n_bytes); + free(data); + return py_bytes; + }); +} + + +void PyClient::delete_bytes(const std::string& name) +{ + MAKE_CLIENT_API({ + _client->delete_bytes(name); + }); +} + + void PyClient::delete_tensor(const std::string& name) { MAKE_CLIENT_API({ @@ -492,6 +529,13 @@ bool PyClient::dataset_exists(const std::string& name) }); } +bool PyClient::bytes_exists(const std::string& name) +{ + return MAKE_CLIENT_API({ + return this->_client->bytes_exists(name); + }); +} + bool PyClient::poll_tensor(const std::string& name, int poll_frequency_ms, int num_tries) @@ -519,6 +563,15 @@ bool PyClient::poll_model(const std::string& name, }); } +bool PyClient::poll_bytes(const std::string& name, + int poll_frequency_ms, + int num_tries) +{ + return MAKE_CLIENT_API({ + return _client->poll_bytes(name, poll_frequency_ms, num_tries); + }); +} + void PyClient::use_tensor_ensemble_prefix(bool use_prefix) { MAKE_CLIENT_API({ @@ -547,6 +600,12 @@ void PyClient::use_list_ensemble_prefix(bool use_prefix) }); } +void PyClient::use_bytes_ensemble_prefix(bool use_prefix) +{ + MAKE_CLIENT_API({ + _client->use_bytes_ensemble_prefix(use_prefix); + }); +} std::vector PyClient::get_db_node_info(std::vector addresses) { diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 1b49008d..6392355a 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -67,6 +67,7 @@ list(APPEND EXECUTABLES client_test_put_get_transpose_3D client_test_put_get_2D client_test_put_get_1D + client_test_put_get_bytes client_test_mnist client_test_mnist_dataset client_test_ensemble diff --git a/tests/cpp/client_test_put_get_bytes.cpp b/tests/cpp/client_test_put_get_bytes.cpp new file mode 100644 index 00000000..e3e9d3a6 --- /dev/null +++ b/tests/cpp/client_test_put_get_bytes.cpp @@ -0,0 +1,71 @@ +/* + * BSD 2-Clause License + * + * Copyright (c) 2021-2024, Hewlett Packard Enterprise + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "client.h" +#include "client_test_utils.h" +#include +#include + +int main(int argc, char* argv[]) { + + // Initialize some byte data + size_t n_bytes = 255; + std::vector input_bytes(n_bytes); + for(size_t i = 0; i < n_bytes; i++) { + input_bytes[i] = i; + } + + SmartRedis::Client client("client_test_put_get_bytes"); + + std::string key = "put_get_bytes_test"; + + client.put_bytes(key, input_bytes.data(), n_bytes); + + if(!client.key_exists(get_prefix() + key)) + throw std::runtime_error("The key does not exist in the database."); + + std::vector output_bytes(n_bytes, 0); + size_t output_n_bytes = 0; + client.unpack_bytes(key, output_bytes.data(), n_bytes, output_n_bytes); + + if (output_n_bytes != n_bytes) { + std::cout<<"Output byte size "<(retrieved_bytes_values); + CHECK((r_pointer)[i] == bytes_value[i]); + } + + free(retrieved_bytes_values); + } + + THEN("The bytes can be retrieved by clients.unpack_bytes()") + { + size_t retrieved_n_bytes = 0; + // Allocate more memory than needed to test unequal buffers + unsigned char* retrieved_bytes_values = (unsigned char*)malloc(n_bytes*2); + + client.unpack_bytes(bytes_name, + retrieved_bytes_values, + n_bytes, + retrieved_n_bytes); + + CHECK(retrieved_bytes_values != NULL); + CHECK(retrieved_n_bytes == n_bytes); + + for(int i = 0; i < n_bytes; i++) { + CHECK((retrieved_bytes_values)[i] == bytes_value[i]); + } + + free(retrieved_bytes_values); + } + + AND_WHEN("The bytes are removed from the database") + { + client.delete_bytes(bytes_name); + + THEN("The bytes no longer exist in the database") + { + CHECK(client.bytes_exists(bytes_name) == false); + } + + THEN("Polling for the bytes returns false") + { + CHECK(client.poll_bytes(bytes_name, 2, 5) == false); + } + } + } + } + log_data(context, LLDebug, "***End Client tensor testing***"); +} + SCENARIO("Testing INFO Functions on Client Object", "[Client]") { std::cout << std::to_string(get_time_offset()) << ": Testing INFO Functions on Client Object" << std::endl; diff --git a/tests/cpp/unit-tests/test_client_prefixing.cpp b/tests/cpp/unit-tests/test_client_prefixing.cpp index 116c3521..36705c03 100644 --- a/tests/cpp/unit-tests/test_client_prefixing.cpp +++ b/tests/cpp/unit-tests/test_client_prefixing.cpp @@ -60,38 +60,111 @@ SCENARIO("Testing Client prefixing") std::string dataset_tensor_key = "dataset_tensor"; std::string dataset_key = "test_dataset"; std::string tensor_key = "test_tensor"; + std::string bytes_key = "test_bytes"; SRTensorType g_type; std::vector g_dims; void* g_result; - THEN("Client prefixing can be tested") + THEN("Client prefixing can be used for tensors") { + // Set env vars to enable key prefixing setenv("SSKEYIN", keyin_env_put, (old_keyin != NULL)); setenv("SSKEYOUT", keyout_env_put, (old_keyout != NULL)); + + // Initialize a client Client client(context); - client.use_dataset_ensemble_prefix(true); + + // Enable prefixing for tensors client.use_tensor_ensemble_prefix(true); + // Create tensor data float* array = (float*)malloc(dims[0]*sizeof(float)); for(int i=0; i