Added support for Base64-encoding/decoding in the HTTP frontend

The new feature applies to both query API and the table ingest API of the frontend's REST service.
lsst · May 12, 2024 · 5a6b6ba · 5a6b6ba
1 parent ade3f06
commit 5a6b6ba
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 5 deletions.
diff --git a/src/czar/HttpCzarIngestModule.cc b/src/czar/HttpCzarIngestModule.cc
@@ -139,7 +139,7 @@ json HttpCzarIngestModule::executeImpl(string const& subModuleName) {
 
 json HttpCzarIngestModule::_ingestData() {
     debug(__func__);
-    checkApiVersion(__func__, 34);
+    checkApiVersion(__func__, 35);
 
     auto const databaseName = body().required<string>("database");
     auto const tableName = body().required<string>("table");

diff --git a/src/czar/HttpCzarQueryModule.cc b/src/czar/HttpCzarQueryModule.cc
@@ -81,7 +81,7 @@ json HttpCzarQueryModule::executeImpl(string const& subModuleName) {
 
 json HttpCzarQueryModule::_submit() {
     debug(__func__);
-    checkApiVersion(__func__, 33);
+    checkApiVersion(__func__, 35);
 
     string const binaryEncodingStr = body().optional<string>("binary_encoding", "hex");
     http::BinaryEncodingMode const binaryEncoding = http::parseBinaryEncoding(binaryEncodingStr);
@@ -138,7 +138,7 @@ json HttpCzarQueryModule::_status() {
 
 json HttpCzarQueryModule::_result() {
     debug(__func__);
-    checkApiVersion(__func__, 33);
+    checkApiVersion(__func__, 35);
     string const binaryEncodingStr = query().optionalString("binary_encoding", "hex");
     http::BinaryEncodingMode const binaryEncoding = http::parseBinaryEncoding(binaryEncodingStr);
     debug(__func__, "binary_encoding=" + http::binaryEncoding2string(binaryEncoding));
@@ -302,6 +302,9 @@ json HttpCzarQueryModule::_rowsToJson(sql::SqlResults& results, json const& sche
                         case http::BinaryEncodingMode::HEX:
                             rowJson.push_back(util::String::toHex(row[i].first, row[i].second));
                             break;
+                        case http::BinaryEncodingMode::B64:
+                            rowJson.push_back(util::String::toBase64(row[i].first, row[i].second));
+                            break;
                         case http::BinaryEncodingMode::ARRAY:
                             // Notes on the std::u8string type and constructor:
                             // 1. This string type is required for encoding binary data which is only possible

diff --git a/src/http/BinaryEncoding.cc b/src/http/BinaryEncoding.cc
@@ -32,6 +32,8 @@ namespace lsst::qserv::http {
 BinaryEncodingMode parseBinaryEncoding(string const& str) {
     if (str == "hex")
         return BinaryEncodingMode::HEX;
+    else if (str == "b64")
+        return BinaryEncodingMode::B64;
     else if (str == "array")
         return BinaryEncodingMode::ARRAY;
     throw invalid_argument("http::" + string(__func__) + " unsupported mode '" + str + "'");
@@ -41,6 +43,8 @@ string binaryEncoding2string(BinaryEncodingMode mode) {
     switch (mode) {
         case BinaryEncodingMode::HEX:
             return "hex";
+        case BinaryEncodingMode::B64:
+            return "b64";
         case BinaryEncodingMode::ARRAY:
             return "array";
     }

diff --git a/src/http/BinaryEncoding.h b/src/http/BinaryEncoding.h
@@ -29,11 +29,12 @@
 namespace lsst::qserv::http {
 
 /// The names of the allowed modes.
-static std::vector<std::string> const allowedBinaryEncodingModes = {"hex", "array"};
+static std::vector<std::string> const allowedBinaryEncodingModes = {"hex", "b64", "array"};
 
 /// Options for encoding data of the binary columns in the JSON result.
 enum class BinaryEncodingMode : int {
     HEX,   ///< The hexadecimal representation stored as a string
+    B64,   ///< Data encoded using Base64 algorithm (with padding as needed)
     ARRAY  ///< JSON array of 8-bit unsigned integers in a range of 0 .. 255.
 };
 

diff --git a/src/replica/ingest/IngestDataHttpSvcMod.cc b/src/replica/ingest/IngestDataHttpSvcMod.cc
@@ -92,7 +92,7 @@ json IngestDataHttpSvcMod::executeImpl(string const& subModuleName) {
 
 json IngestDataHttpSvcMod::_syncProcessData() {
     debug(__func__);
-    checkApiVersion(__func__, 34);
+    checkApiVersion(__func__, 35);
 
     auto const context_ = context() + __func__;
     auto const config = serviceProvider()->config();
@@ -258,6 +258,9 @@ json IngestDataHttpSvcMod::_syncProcessData() {
                     case http::BinaryEncodingMode::HEX:
                         row.append(_translateHexString(context_, jsonColumn, rowIdx, colIdx));
                         break;
+                    case http::BinaryEncodingMode::B64:
+                        row.append(_translateBase64String(context_, jsonColumn, rowIdx, colIdx));
+                        break;
                     case http::BinaryEncodingMode::ARRAY: {
                         u8string const str = _translateByteArray(context_, jsonColumn, rowIdx, colIdx);
                         row.append(reinterpret_cast<char const*>(str.data()), str.size());
@@ -333,6 +336,28 @@ string IngestDataHttpSvcMod::_translateHexString(string const& context_, json co
     throw http::Error(context_, _contrib.error);
 }
 
+string IngestDataHttpSvcMod::_translateBase64String(string const& context_, json const& jsonColumn,
+                                                    size_t rowIdx, size_t colIdx) {
+    if (jsonColumn.is_string()) {
+        try {
+            return util::String::fromBase64(jsonColumn.get<string>());
+        } catch (exception const& ex) {
+            _contrib.error = "failed to decode a value of the '" +
+                             http::binaryEncoding2string(http::BinaryEncodingMode::B64) +
+                             "' binary encoded column at row " + to_string(rowIdx) + " and column " +
+                             to_string(colIdx) + ", ex: " + string(ex.what());
+        }
+    } else {
+        _contrib.error = "unsupported type name '" + string(jsonColumn.type_name()) + "' found at row " +
+                         to_string(rowIdx) + " and column " + to_string(colIdx) +
+                         " where the string type was expected";
+    }
+    bool const failed = true;
+    _contrib = serviceProvider()->databaseServices()->startedTransactionContrib(_contrib, failed);
+    _failed(context_);
+    throw http::Error(context_, _contrib.error);
+}
+
 u8string IngestDataHttpSvcMod::_translateByteArray(string const& context_, json const& jsonColumn,
                                                    size_t rowIdx, size_t colIdx) {
     if (jsonColumn.is_array()) {

diff --git a/src/replica/ingest/IngestDataHttpSvcMod.h b/src/replica/ingest/IngestDataHttpSvcMod.h
@@ -101,6 +101,8 @@ class IngestDataHttpSvcMod : public http::ModuleBase, public IngestFileSvc {
 
     std::string _translateHexString(std::string const& context_, nlohmann::json const& jsonColumn,
                                     size_t rowIdx, size_t colIdx);
+    std::string _translateBase64String(std::string const& context_, nlohmann::json const& jsonColumn,
+                                       size_t rowIdx, size_t colIdx);
     std::u8string _translateByteArray(std::string const& context_, nlohmann::json const& jsonColumn,
                                       size_t rowIdx, size_t colIdx);
     std::string _translatePrimitiveType(std::string const& context_, nlohmann::json const& jsonColumn,