Enable putting bytes into the database via C++ and Python (#521)

This PR adds client API methods in C++ and Python 
for sending and receiving byte strings from the database.
Methods for put, get, unpack, poll, exists, and delete have 
been implemented.

[ committed by @mellis13 ]
[ reviewed by @ashao ]

Author: mellis13

doc/changelog.md

@@ -6,10 +6,20 @@ To be released at a future time.
 
 Description
 
+-  Add Client API functions to put, get, unpack,
+   delete, poll, and check for existance of raw bytes for the
+   C++ and Python clients.
+-  Fix lint issues using make lint
 -  Reenable move semantics and fix compiler warnings.
 
 Detailed Notes
 
+-  Add Client API functions to put, get, unpack,
+   delete, poll, and check for existance of raw bytes for the
+   C++ and Python clients.
+   ([PR521](https://github.com/CrayLabs/SmartRedis/pull/521))
+-  Fix lint issues using make lint
+   ([PR522](https://github.com/CrayLabs/SmartRedis/pull/522))
 -  Fix compiler warnings stemming from override and const
    keywords as well as move semantics impliclty disabled
    because of std::random_device.
@@ -21,14 +31,11 @@ Released on 27 September, 2024
 
 Description
 
--  Fix lint issues using make lint
 -  Fix RedisAI build to allow for compilation with GCC-14
 -  Fix a memory leak in the Fortran Dataset implementation
 
 Detailed Notes
 
--  Fix lint issues using make lint
-   ([PR522](https://github.com/CrayLabs/SmartRedis/pull/522))
 -  Fix RedisAI build to allow for compilation with GCC-14. Also,
    we only use the Torch backend and change the compilation of
    RedisAI to use CMake (like SmartSim)

examples/serial/cpp/CMakeLists.txt

@@ -50,6 +50,7 @@ list(APPEND EXECUTABLES
     smartredis_dataset
     smartredis_model
     smartredis_mnist
+    smartredis_put_get_bytes
 )
 
 # Build the examples

examples/serial/cpp/smartredis_put_get_bytes.cpp

@@ -0,0 +1,69 @@
+/*
+ * BSD 2-Clause License
+ *
+ * Copyright (c) 2021-2024, Hewlett Packard Enterprise
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "client.h"
+#include <vector>
+#include <string>
+
+int main(int argc, char* argv[]) {
+
+  // Initialize some byte data
+  size_t n_bytes = 255;
+  std::vector<char> input_bytes(n_bytes);
+  for(size_t i = 0; i < n_bytes; i++) {
+    input_bytes[i] = i;
+  }
+
+  SmartRedis::Client client("client_test_put_get_bytes");
+
+  std::string key = "put_get_bytes_test";
+
+  client.put_bytes(key, input_bytes.data(), n_bytes);
+
+  std::vector<char> output_bytes(n_bytes, 0);
+
+  size_t received_bytes = 0;
+  client.unpack_bytes(key, output_bytes.data(), n_bytes, 
+                      received_bytes);
+
+    if (received_bytes != n_bytes) {
+      std::cout<<"Output byte size "<<received_bytes<<" does not match."<<std::endl;
+        throw std::exception();
+    }
+
+  for(int i = 0; i < n_bytes; i++) {
+    if (output_bytes[i] != input_bytes[i]) {
+      std::cout<<"Byte "<<i<<" does not match."<<std::endl;
+      throw std::exception();
+    }
+  }
+
+  std::cout<<"Put bytes test complete"<<std::endl;
+
+  return 0;
+}

include/client.h

@@ -223,6 +223,77 @@ class Client : public SRObject
                         const SRTensorType type,
                         const SRMemoryLayout mem_layout);
 
+        /*!
+        *   \brief Puts a byte string into the database
+        *   \details The provided byte string is placed into the database
+        *            without any type associated with the bytes.  
+        *            The final key under which the bytes are stored
+        *            may be formed by applying a prefix to the supplied
+        *            name. See use_bytes_ensemble_prefix() for
+        *            more details.
+        *   \param name The name for referencing the bytes in the database
+        *   \param data The data
+        *   \param n_bytes The number of bytes in the provided data
+        *   \throw SmartRedis::Exception if put bytes command fails
+        */
+        void put_bytes(const std::string& name,
+                       const void* bytes,
+                       const size_t n_bytes);
+
+        /*!
+        *   \brief Retrieve bytes from the database and place into a new
+        *          memory buffer
+        *   \details The key used to locate the stored bytes
+        *            may be formed by applying a prefix to the supplied
+        *            name. See set_data_source()
+        *            and use_bytes_ensemble_prefix() for more details.
+        *            Unlike other "get" operations in the client, the
+        *            the byte data memeory is allocated but not managed
+        *            by the client.  The caller is responsible for invoking
+        *            free() on the returned memory.
+        *   \param name The name for referencing the bytes in the database
+        *   \param data A user-provided pointer that will be modified to
+        *               point to a new block of memory containing the bytes
+        *   \param n_bytes A user-provided size_t variable that will be modified
+        *                  to indicate the total number of bytes.
+        *   \throw SmartRedis::Exception if get bytes command fails
+        */
+        void get_bytes(const std::string& name,
+                       void*& data,
+                       size_t& n_bytes);
+
+        /*!
+        *   \brief Retrieve bytes from the database and place into memory
+        *          provided by the caller
+        *   \details The key used to locate the stored bytes
+        *            may be formed by applying a prefix to the supplied
+        *            name. See set_data_source()
+        *            and use_bytes_ensemble_prefix() for more details.
+        *   \param name The name for referencing the bytes in the database
+        *   \param data A buffer into which to place byte data
+        *   \param n_bytes The number of bytes in the provided memory block
+        *   \param n_used_bytes The number of bytes in the provided 
+        *                       buffer that were used
+        *   \throw SmartRedis::Exception if unpack bytes command fails or
+        *          the number of provided bytes is less than the retrieved
+        *          number of bytes
+        */
+        void unpack_bytes(const std::string& name,
+                          void* data,
+                          const size_t n_bytes,
+                          size_t& n_used_bytes);
+
+        /*!
+        *   \brief Delete bytes from the database
+        *   \details The key used to locate the bytes to be
+        *            deleted may be formed by applying a prefix to the
+        *            supplied name. See set_data_source()
+        *            and use_bytes_ensemble_prefix() for more details.
+        *   \param name The name of the bytes to delete
+        *   \throw SmartRedis::Exception if delete bytes command fails
+        */
+        void delete_bytes(const std::string& name);
+
         /*!
         *   \brief Retrieve the tensor data, dimensions, and type for the
         *          provided tensor key. This function will allocate and retain
@@ -795,18 +866,6 @@ class Client : public SRObject
         */
         bool key_exists(const std::string& key);
 
-        /*!
-        *   \brief Check if a model (or script) key exists in the database
-        *   \details The model or script key used to check for existence
-        *            may be formed by applying a prefix to the supplied
-        *            name. See set_data_source()
-        *            and use_model_ensemble_prefix() for more details.
-        *   \param name The model/script name to be checked in the database
-        *   \returns Returns true if the model exists in the database
-        *   \throw SmartRedis::Exception if model exists command fails
-        */
-        bool model_exists(const std::string& name);
-
         /*!
         *   \brief Check if a tensor key exists in the database
         *   \details The tensor key used to check for existence
@@ -831,6 +890,30 @@ class Client : public SRObject
         */
         bool dataset_exists(const std::string& name);
 
+        /*!
+        *   \brief Check if a model (or script) key exists in the database
+        *   \details The model or script key used to check for existence
+        *            may be formed by applying a prefix to the supplied
+        *            name. See set_data_source()
+        *            and use_model_ensemble_prefix() for more details.
+        *   \param name The model/script name to be checked in the database
+        *   \returns Returns true if the model exists in the database
+        *   \throw SmartRedis::Exception if model exists command fails
+        */
+        bool model_exists(const std::string& name);
+
+        /*!
+        *   \brief Check if bytes exists in the database
+        *   \details The key used to check for existence
+        *            may be formed by applying a prefix to the supplied
+        *            name. See set_data_source()
+        *            and use_bytes_ensemble_prefix() for more details.
+        *   \param name The bytes name to be checked in the database
+        *   \returns Returns true if the bytes exists in the database
+        *   \throw SmartRedis::Exception if bytes exists command fails
+        */
+        bool bytes_exists(const std::string& name);
+
         /*!
         *   \brief Check if a key exists in the database, repeating the check
         *          at a specified polling interval
@@ -903,6 +986,25 @@ class Client : public SRObject
                         int poll_frequency_ms,
                         int num_tries);
 
+        /*!
+        *   \brief Check if bytes exists in the database, repeating
+        *          the check at a specified polling interval
+        *   \details The key used to check for existence
+        *            may be formed by applying a prefix to the supplied
+        *            name. See set_data_source()
+        *            and use_bytes_ensemble_prefix() for more details.
+        *   \param name The bytes name to be checked in the database
+        *   \param poll_frequency_ms The time delay between checks,
+        *                            in milliseconds
+        *   \param num_tries The total number of times to check for the name
+        *   \returns Returns true if the bytes is found within the
+        *            specified number of tries, otherwise false.
+        *   \throw SmartRedis::Exception if poll bytes command fails
+        */
+        bool poll_bytes(const std::string& name,
+                        int poll_frequency_ms,
+                        int num_tries);
+
         /*!
         *   \brief Set the data source, a key prefix for future operations.
         *   \details When running multiple applications, such as an ensemble
@@ -1011,6 +1113,25 @@ class Client : public SRObject
         */
         void use_list_ensemble_prefix(bool use_prefix);
 
+        /*!
+        *   \brief Control whether bytes are prefixed
+        *   \details This function can be used to avoid key collisions in an
+        *            ensemble by prepending the string value from the
+        *            environment variable SSKEYIN and/or SSKEYOUT to
+        *            raw byte names.  Prefixes will only be used if
+        *            they were previously set through the environment variables
+        *            SSKEYOUT and SSKEYIN. Keys for raw bytes created
+        *            before this function is called will not be retroactively
+        *            prefixed. By default, the client prefixes raw bytes
+        *            keys with the first prefix specified with the SSKEYIN
+        *            and SSKEYOUT environment variables.
+        *  \param use_prefix If set to true, all future operations
+        *                    on raw bytes will use a prefix, if available.
+        *  \throw SmartRedis::Exception for failed activation of
+        *         raw byte prefixing
+        */
+        void use_bytes_ensemble_prefix(bool use_prefix);
+
         /*!
         *   \brief Returns information about the given database node
         *   \param address The address of the database node (host:port)
@@ -1569,6 +1690,12 @@ class Client : public SRObject
         */
         bool _use_list_prefix;
 
+        /*!
+        * \brief Flag determining whether prefixes should be used
+        *        for raw byte keys.
+        */
+        bool _use_bytes_prefix;
+
         /*!
         * \brief Our configuration options, used to access runtime settings
         */
@@ -1664,6 +1791,15 @@ class Client : public SRObject
         */
         inline std::string _build_list_key(const std::string& list_name,
                                            bool on_db);
+        /*!
+        * \brief Build full formatted key for bytes, based on
+        *        current prefix settings.
+        * \param name Unprefixed bytes name
+        * \param on_db Indicates whether the key refers to an entity
+        *              which is already in the database.
+        */
+        inline std::string _build_bytes_key(const std::string& name,
+                                            bool on_db);
 
         /*!
         *   \brief Append the Command associated with