From 57536535da44db7bb0dfbcf73741d0b417af62a7 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Fri, 28 Nov 2025 12:11:27 +0100 Subject: [PATCH 01/14] New delta-sharing node-type Signed-off-by: Ritesh.K --- CMakeLists.txt | 5 + .../nodes/delta_sharing/delta_sharing.hpp | 67 +++ .../delta_sharing/delta_sharing_client.hpp | 38 ++ .../delta_sharing_rest_client.hpp | 44 ++ .../villas/nodes/delta_sharing/functions.hpp | 27 ++ .../villas/nodes/delta_sharing/protocol.hpp | 212 +++++++++ lib/nodes/CMakeLists.txt | 8 + lib/nodes/delta_sharing/delta_sharing.cpp | 413 ++++++++++++++++++ .../delta_sharing/delta_sharing_client.cpp | 170 +++++++ .../delta_sharing_rest_client.cpp | 272 ++++++++++++ lib/nodes/delta_sharing/functions.cpp | 87 ++++ packaging/deps.sh | 50 +++ 12 files changed, 1393 insertions(+) create mode 100644 include/villas/nodes/delta_sharing/delta_sharing.hpp create mode 100644 include/villas/nodes/delta_sharing/delta_sharing_client.hpp create mode 100644 include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp create mode 100644 include/villas/nodes/delta_sharing/functions.hpp create mode 100644 include/villas/nodes/delta_sharing/protocol.hpp create mode 100644 lib/nodes/delta_sharing/delta_sharing.cpp create mode 100644 lib/nodes/delta_sharing/delta_sharing_client.cpp create mode 100644 lib/nodes/delta_sharing/delta_sharing_rest_client.cpp create mode 100644 lib/nodes/delta_sharing/functions.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cb64b0cce..3b3b21ffd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,10 @@ pkg_check_modules(CGRAPH IMPORTED_TARGET libcgraph>=2.30) pkg_check_modules(GVC IMPORTED_TARGET libgvc>=2.30) pkg_check_modules(LIBUSB IMPORTED_TARGET libusb-1.0>=1.0.23) pkg_check_modules(NANOMSG IMPORTED_TARGET nanomsg) +pkg_check_modules(ARROW IMPORTED_TARGET REQUIRED arrow) +pkg_check_modules(PARQUET IMPORTED_TARGET REQUIRED parquet) +# pkg_check_modules(RESTCLIENT IMPORTED_TARGET REQUIRED restclient-cpp) + if(NOT NANOMSG_FOUND) pkg_check_modules(NANOMSG IMPORTED_TARGET libnanomsg>=1.0.0) endif() @@ -214,6 +218,7 @@ cmake_dependent_option(WITH_NODE_ULDAQ "Build with uldaq node-type" cmake_dependent_option(WITH_NODE_WEBRTC "Build with webrtc node-type" "${WITH_DEFAULTS}" "WITH_WEB; LibDataChannel_FOUND" OFF) cmake_dependent_option(WITH_NODE_WEBSOCKET "Build with websocket node-type" "${WITH_DEFAULTS}" "WITH_WEB" OFF) cmake_dependent_option(WITH_NODE_ZEROMQ "Build with zeromq node-type" "${WITH_DEFAULTS}" "LIBZMQ_FOUND; NOT WITHOUT_GPL" OFF) +cmake_dependent_option(WITH_NODE_DELTASHARING "Build with delta_sharing node-type" "${WITH_DEFAULTS}" "" OFF) # Set a default for the build type if("${CMAKE_BUILD_TYPE}" STREQUAL "") diff --git a/include/villas/nodes/delta_sharing/delta_sharing.hpp b/include/villas/nodes/delta_sharing/delta_sharing.hpp new file mode 100644 index 000000000..d2a424210 --- /dev/null +++ b/include/villas/nodes/delta_sharing/delta_sharing.hpp @@ -0,0 +1,67 @@ +/* Node type: Delta Sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include + +namespace arrow { class Table; } + +namespace villas { +namespace node { + +// Forward declarations +class NodeCompat; + +struct delta_sharing { + // Configuration + std::string profile_path; + std::string cache_dir; + std::string table_path; + size_t batch_size; + std::string schema; + std::string share; + std::string table; + + // Client and state + std::shared_ptr client; + std::shared_ptr> schemas; + std::shared_ptr table_ptr; + std::shared_ptr> tables; + std::shared_ptr> shares; + + enum class TableOp { TABLE_NOOP, TABLE_READ, TABLE_WRITE } table_op; +}; + +char *deltaSharing_print(NodeCompat *n); + +int deltaSharing_parse(NodeCompat *n, json_t *json); + +int deltaSharing_start(NodeCompat *n); + +int deltaSharing_stop(NodeCompat *n); + +int deltaSharing_init(NodeCompat *n); + +int deltaSharing_destroy(NodeCompat *n); + +int deltaSharing_poll_fds(NodeCompat *n, int fds[]); + +int deltaSharing_read(NodeCompat *n, struct Sample *const smps[], unsigned cnt); + +int deltaSharing_write(NodeCompat *n, struct Sample *const smps[], unsigned cnt); + +} // namespace node +} // namespace villas diff --git a/include/villas/nodes/delta_sharing/delta_sharing_client.hpp b/include/villas/nodes/delta_sharing/delta_sharing_client.hpp new file mode 100644 index 000000000..1eeff28ad --- /dev/null +++ b/include/villas/nodes/delta_sharing/delta_sharing_client.hpp @@ -0,0 +1,38 @@ +/* Node type: Delta Sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +namespace DeltaSharing +{ + + struct DeltaSharingClient + { + public: + DeltaSharingClient(std::string filename, std::optional cacheLocation); + std::shared_ptr LoadAsArrowTable(std::string &url); + std::shared_ptr ReadTableFromCache(std::string &url); + const std::shared_ptr> ListShares(int maxResult, std::string pageToken) const; + const std::shared_ptr> ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; + const std::shared_ptr> ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const; + const std::shared_ptr> ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; + const std::shared_ptr> ListFilesInTable(const DeltaSharingProtocol::Table) const; + const DeltaSharingProtocol::Metadata QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; + const int GetNumberOfThreads() { return this->maxThreads; }; + void PopulateCache(std::string url) { this->restClient.PopulateCache(url,this->cacheLocation); }; + protected: + private: + DeltaSharingRestClient restClient; + std::string cacheLocation; + int maxThreads; + }; +}; diff --git a/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp b/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp new file mode 100644 index 000000000..2045d6999 --- /dev/null +++ b/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp @@ -0,0 +1,44 @@ +/* Node type: Delta Sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +namespace DeltaSharing +{ + struct DeltaSharingRestClient + { + public: + DeltaSharingRestClient(std::string filename); + ~DeltaSharingRestClient(); + const std::shared_ptr> ListShares(int maxResult, std::string pageToken) const; + const std::shared_ptr> ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; + const std::shared_ptr> ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const; + const std::shared_ptr> ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; + const std::shared_ptr> ListFilesInTable(const DeltaSharingProtocol::Table) const; + const DeltaSharingProtocol::Metadata QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; + const DeltaSharingProtocol::DeltaSharingProfile &GetProfile() const; + RestClient::Response get(std::string url); + void PopulateCache(std::string url, std::string cacheLocation); + const bool shouldRetry(RestClient::Response &response) const; + + protected: + json ReadFromFile(std::string filename); + + private: + DeltaSharingProtocol::DeltaSharingProfile profile; + static const std::string user_agent; + }; +}; diff --git a/include/villas/nodes/delta_sharing/functions.hpp b/include/villas/nodes/delta_sharing/functions.hpp new file mode 100644 index 000000000..3d1cd0d30 --- /dev/null +++ b/include/villas/nodes/delta_sharing/functions.hpp @@ -0,0 +1,27 @@ +/* Node type: Delta Sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace DeltaSharing { + +const std::vector ParseURL(std::string path); +std::shared_ptr +NewDeltaSharingClient(std::string profile, + std::optional cacheLocation); +const std::shared_ptr LoadAsArrowTable(std::string path, + int fileno); +}; + +// namespace DeltaSharing diff --git a/include/villas/nodes/delta_sharing/protocol.hpp b/include/villas/nodes/delta_sharing/protocol.hpp new file mode 100644 index 000000000..1814431ac --- /dev/null +++ b/include/villas/nodes/delta_sharing/protocol.hpp @@ -0,0 +1,212 @@ +/* Node type: Delta Sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace DeltaSharing +{ + + namespace DeltaSharingProtocol + { + + using json = nlohmann::json; + + struct DeltaSharingProfile + { + public: + int shareCredentialsVersion; + std::string endpoint; + std::string bearerToken; + std::optional expirationTime; + }; + + inline void from_json(const json &j, DeltaSharingProfile &p) + { + if (j.contains("shareCredentialsVersion")) + { + p.shareCredentialsVersion = j["shareCredentialsVersion"]; + } + if (j.contains("endpoint")) + { + p.endpoint = j["endpoint"]; + } + if (j.contains("bearerToken")) + { + p.bearerToken = j["bearerToken"]; + } + if (j.contains("expirationTime")) + { + p.expirationTime = j["expirationTime"]; + } + }; + + struct Share + { + public: + std::string name = ""; + std::optional id; + }; + + inline void from_json(const json &j, Share &s) + { + s.name = j["name"]; + if (j.contains("id") == false) + { + s.id = std::nullopt; + } + else + { + s.id = j["id"]; + } + }; + + struct Schema + { + public: + std::string name; + std::string share; + }; + + inline void from_json(const json &j, Schema &s) + { + s.name = j["name"]; + if (j.contains("share") == true) + { + s.share = j["share"]; + } + }; + + struct Table + { + public: + std::string name; + std::string share; + std::string schema; + }; + + inline void from_json(const json &j, Table &t) + { + if (j.contains("name")) + { + t.name = j["name"]; + } + if (j.contains("share")) + { + t.share = j["share"]; + } + if (j.contains("schema")) + { + t.schema = j["schema"]; + } + }; + + struct File + { + public: + std::string url; + std::optional id; + std::map partitionValues; + std::size_t size; + std::string stats; + std::optional timestamp; + std::optional version; + }; + + inline void from_json(const json &j, File &f) + { + if (j.contains("url")) + { + f.url = j["url"]; + } + if (j.contains("id")) + { + f.id = j["id"]; + } + if (j.contains("partitionValues")) + { + json arr = j["partitionValues"]; + auto f2 = f.partitionValues; + if (arr.is_array()) + { + for (auto it = arr.begin(); it < arr.end(); it++) + { + f2.insert({it.key(), it.value()}); + } + } + } + if (j.contains("size")) + { + f.size = j["size"]; + } + if (j.contains("stats")) + { + f.stats = j["stats"]; + } + if (j.contains("timestamp")) + { + f.timestamp = j["timestamp"]; + } + if (j.contains("version")) + { + f.version = j["version"]; + } + }; + + struct Format { + std::string provider; + + }; + + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Format, provider) + + struct Metadata + { + Format format; + std::string id; + std::vector partitionColumns; + std::string schemaString; + }; + + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Metadata,format, id, partitionColumns,schemaString) + + struct data + { + std::vector predicateHints; + int limitHint; + }; + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(data, predicateHints, limitHint) + + struct format + { + std::string provider; + }; + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(format, provider) + + struct protocol + { + int minReaderVersion; + }; + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(protocol,minReaderVersion) + + struct stats + { + long long numRecords; + long minValues; + long maxValues; + long nullCount; + }; + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(stats, numRecords, minValues, maxValues, nullCount) + + }; +}; diff --git a/lib/nodes/CMakeLists.txt b/lib/nodes/CMakeLists.txt index 068ad9fba..1b6a640fa 100644 --- a/lib/nodes/CMakeLists.txt +++ b/lib/nodes/CMakeLists.txt @@ -202,6 +202,14 @@ if(WITH_NODE_WEBRTC) list(APPEND LIBRARIES LibDataChannel::LibDataChannel) endif() +# Enable Delta Sharing +if(WITH_NODE_DELTASHARING) + list(APPEND NODE_SRC delta_sharing/delta_sharing.cpp delta_sharing/delta_sharing_client.cpp + delta_sharing/delta_sharing_rest_client.cpp delta_sharing/functions.cpp + ) + list(APPEND LIBRARIES ${ARROW_LIBRARIES} ${PARQUET_LIBRARIES} ${RESTCLIENT_LIBRARIES}) +endif() + add_library(nodes STATIC ${NODE_SRC}) target_include_directories(nodes PUBLIC ${INCLUDE_DIRS}) target_link_libraries(nodes PUBLIC ${LIBRARIES}) diff --git a/lib/nodes/delta_sharing/delta_sharing.cpp b/lib/nodes/delta_sharing/delta_sharing.cpp new file mode 100644 index 000000000..2a2b6f8ea --- /dev/null +++ b/lib/nodes/delta_sharing/delta_sharing.cpp @@ -0,0 +1,413 @@ +/* Node type: delta_sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace villas; +using namespace villas::node; + +static const char *const OP_READ = "read"; +static const char *const OP_WRITE = "write"; +static const char *const OP_NOOP = "noop"; + +int villas::node::deltaSharing_parse(NodeCompat *n, json_t *json) { + auto *d = n->getData(); + + int ret; + json_error_t err; + + const char *profile_path = nullptr; + const char *cache_dir = nullptr; + const char *table_path = nullptr; + const char *op = nullptr; + const char *schema = nullptr; + const char *share = nullptr; + const char *table = nullptr; + int batch_size = 0; + + ret = json_unpack_ex(json, &err, 0, + "{ s?: s, s?: s, s?: s, s?: s, s?: s, s?: s, s?: s, s?: i }", + "profile_path", &profile_path, "schema", &schema, + "share", &share, "table", &table,"cache_dir", &cache_dir, "table_path", + &table_path, "op", &op, "batch_size", &batch_size); + + if (ret) + throw ConfigError(json, err, "node-config-node-delta_sharing"); + + if (profile_path) + d->profile_path = profile_path; + if (share) + d->share = share; + if (schema) + d->schema = schema; + if (table) + d->table = table; + if (cache_dir) + d->cache_dir = cache_dir; + if (table_path) + d->table_path = table_path; + if (batch_size > 0) + d->batch_size = static_cast(batch_size); + + if (op) { + if (strcmp(op, OP_READ) == 0) + d->table_op = delta_sharing::TableOp::TABLE_READ; + else if (strcmp(op, OP_WRITE) == 0) + d->table_op = delta_sharing::TableOp::TABLE_WRITE; + else + d->table_op = delta_sharing::TableOp::TABLE_NOOP; + } + + return 0; +} + +char *villas::node::deltaSharing_print(NodeCompat *n) { + auto *d = n->getData(); + + std::string info = + std::string("profile_path=") + d->profile_path + ", share =" + d->share + + ", schema =" + d->schema + ", table =" + d->table + + ", cache_dir=" + d->cache_dir + ", table_path=" + d->table_path + + ", op=" + + (d->table_op == delta_sharing::TableOp::TABLE_READ + ? OP_READ + : (d->table_op == delta_sharing::TableOp::TABLE_WRITE ? OP_WRITE + : OP_NOOP)); + + return strdup(info.c_str()); +} + + +int villas::node::deltaSharing_start(NodeCompat *n) { + auto *d = n->getData(); + + if (d->profile_path.empty()) + throw RuntimeError( + "'profile_path' must be configured for delta_sharing node"); + + std::optional cache_opt = + d->cache_dir.empty() ? std::nullopt + : std::optional(d->cache_dir); + + d->client = DeltaSharing::NewDeltaSharingClient(d->profile_path, cache_opt); + + if (!d->client) + throw RuntimeError("Failed to create Delta Sharing client"); + + //List all shares from the profile path + d->shares = d->client->ListShares(100, ""); + + const auto &shares = *d->shares; + + for (const auto &share : shares) { + n->logger->info("Listing share {}", share.name); + d->schemas = d->client->ListSchemas(share, 100, ""); + //List all tables in a share + d->tables = d->client->ListAllTables(share, 100, ""); + //Check if tables are fetched correctly + n->logger->info("Table 1 {}", d->tables->at(0).name); + + } + + return 0; +} + +int villas::node::deltaSharing_stop(NodeCompat *n) { + auto *d = n->getData(); + d->table_ptr.reset(); + d->tables.reset(); + d->shares.reset(); + d->client.reset(); + return 0; +} + +int villas::node::deltaSharing_init(NodeCompat *n) { + auto *d = n->getData(); + + // d->profile_path = ""; + // d->cache_dir = ""; + // d->table_path = ""; + d->batch_size = 0; + + d->client.reset(); + d->table_ptr.reset(); + d->tables.reset(); + d->shares.reset(); + d->table_op = delta_sharing::TableOp::TABLE_NOOP; + n->logger->info("Init for Delta Sharing node"); + + return 0; +} + +int villas::node::deltaSharing_destroy(NodeCompat *n) { + auto *d = n->getData(); + d->client.reset(); + if (d->table_ptr != NULL) + d->table_ptr.reset(); + if (d->tables != NULL) + d->tables.reset(); + if (d->shares != NULL) + d->shares.reset(); + return 0; +} + +int villas::node::deltaSharing_poll_fds(NodeCompat *n, int fds[]) { + (void)n; + (void)fds; + return -1; // no polling support +} + +int villas::node::deltaSharing_read(NodeCompat *n, struct Sample *const smps[], + unsigned cnt) { + + auto *d = n->getData(); + + if (!d->client) { + n->logger->error("Delta Sharing client not initialized"); + return -1; + } + + if (d->table_path.empty()) { + n->logger->error("No table path configured"); + return -1; + } + + try { + auto path = DeltaSharing::ParseURL(d->table_path); + + if (path.size() != 4) { + n->logger->error("Invalid table path format. Expected: server#share.schema.table"); + return -1; + } + + DeltaSharing::DeltaSharingProtocol::Table table; + table.share = path[1]; + table.schema = path[2]; + table.name = path[3]; + + //Get files in the table + auto files = d->client->ListFilesInTable(table); + if (!files || files->empty()) { + n->logger->info("No files found in table"); + return 0; + } + + for (const auto &f : *files) { + d->client->PopulateCache(f.url); + } + + //Load the first file as an Arrow table + if (!d->table_ptr) { + d->table_ptr = d->client->LoadAsArrowTable(files->at(0).url); + + if (!d->table_ptr) { + n->logger->error("Failed to laod table from Delta Sharing server"); + } + } + + unsigned samples_read = 0; + auto num_rows = d->table_ptr->num_rows(); + unsigned num_cols = d->table_ptr->num_columns(); + + auto signals = n->getInputSignals(false); + if (!signals) { + n->logger->error("No input signals configured"); + return -1; + } + + for (unsigned i = 0; i < cnt && i < num_rows; i++) { + auto *smp = smps[i]; + n->logger->info("Row name {}", d->table_ptr->ColumnNames().at(3)); + smp->length = signals->size(); + smp->capacity = signals->size(); + smp->ts.origin = time_now(); + + for (unsigned col = 0; col < num_cols && col < signals->size(); col++) { + auto chunked_array = d->table_ptr->column(col); + auto first_chunk = chunked_array->chunk(0); + switch (first_chunk->type_id()) { + case arrow::Type::DOUBLE: { + auto double_array = + std::static_pointer_cast(first_chunk); + smp->data[col].f = double_array->Value(i); + break; + } + case arrow::Type::FLOAT: { + auto float_array = + std::static_pointer_cast(first_chunk); + smp->data[col].f = float_array->Value(i); + } + case arrow::Type::INT64: { + auto int_array = std::static_pointer_cast(first_chunk); + smp->data[col].i = int_array->Value(i); + break; + } + case arrow::Type::INT32: { + auto int_array = std::static_pointer_cast(first_chunk); + smp->data[col].i = int_array->Value(i); + break; + } + /* case arrow::Type::STRING: { + auto string_array = + std::static_pointer_cast(first_chunk); + smp->data[col]. + } */ + default: + n->logger->warn("Unsupported data type for column {}", col); + smp->data[col].f = 0.0; + } + } + samples_read++; + } + + n->logger->debug("Read {} samples from Delta Sharing table", samples_read); + return samples_read; + } catch (const std::exception &e) { + n->logger->error("Error reading from Delta Sharing table: {}", e.what()); + return -1; + } + } + +//TODO: write table to delta sharing server. Implementation to be tested +int villas::node::deltaSharing_write(NodeCompat *n, struct Sample *const smps[], + unsigned cnt) { + auto *d = n->getData(); + + if (!d->client) { + n->logger->error("Delta Sharing client not initialized"); + return -1; + } + + if (d->table_path.empty()) { + n->logger->error("No table path configured"); + return -1; + } + + try { + auto path_parts = DeltaSharing::ParseURL(d->table_path); + if (path_parts.size() != 4) { + n->logger->error("Invalid table path format. Expected: server#share.schema.table"); + return -1; + } + + auto signals = n->getOutputSignals(false); + if (!signals) { + n->logger->error("No output signals configured"); + return -1; + } + + std::vector> arrays; + std::vector> fields; + + for (unsigned col = 0; col < signals->size(); col++) { + auto signal = signals->at(col); + + std::string field_name = signal->name; + if (field_name.empty()) { + field_name = "col_" + std::to_string(col); + } + + //Determine arrow data type from signal data type + std::shared_ptr data_type; + switch (signal->type) { + case SignalType::FLOAT: + data_type = arrow::float64(); + break; + case SignalType::INTEGER: + data_type = arrow::int64(); + break; + default: + data_type = arrow::float64(); + } + + fields.push_back(arrow::field(field_name, data_type)); + + //create Arrow array from sampled data + std::shared_ptr array; + switch (signal->type) { + case SignalType::FLOAT: { + std::vector values; + for (unsigned i = 0; i < cnt; i++) { + values.push_back(smps[i]->data[col].f); + } + arrow::DoubleBuilder builder; + PARQUET_THROW_NOT_OK(builder.AppendValues(values)); + PARQUET_THROW_NOT_OK(builder.Finish(&array)); + break; + } + case SignalType::INTEGER: { + std::vector values; + for (unsigned i = 0; i < cnt; i++) { + values.push_back(smps[i]->data[col].i); + } + arrow::Int64Builder builder; + PARQUET_THROW_NOT_OK(builder.AppendValues(values)); + PARQUET_THROW_NOT_OK(builder.Finish(&array)); + break; + } + default: + n->logger->warn("Unsupported signal type for column {}", col); + continue; + } + + arrays.push_back(array); + } + // Create Arrow schema and table + auto schema = std::make_shared(fields); + auto table = arrow::Table::Make(schema, arrays); + + // Store the table for potential future use + d->table_ptr = table; + + n->logger->debug("Wrote {} samples to Delta Sharing table", cnt); + return cnt; + } catch (const std::exception &e) { + n->logger->error("Error writing to Delta Sharing: {}", e.what()); + return -1; + } +} + +static NodeCompatType p; + +__attribute__((constructor(110))) static void register_plugin() { + p.name = "delta_sharing"; + p.description = "Delta Sharing protocol node"; + p.vectorize = 1; + p.size = sizeof(struct delta_sharing); + p.init = deltaSharing_init; + p.destroy = deltaSharing_destroy; + p.parse = deltaSharing_parse; + p.print = deltaSharing_print; + p.start = deltaSharing_start; + p.stop = deltaSharing_stop; + p.read = deltaSharing_read; + p.write = deltaSharing_write; + p.poll_fds = deltaSharing_poll_fds; + + static NodeCompatFactory ncp(&p); +} diff --git a/lib/nodes/delta_sharing/delta_sharing_client.cpp b/lib/nodes/delta_sharing/delta_sharing_client.cpp new file mode 100644 index 000000000..faa13cc5d --- /dev/null +++ b/lib/nodes/delta_sharing/delta_sharing_client.cpp @@ -0,0 +1,170 @@ +/* Node type: delta_sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DeltaSharing +{ + DeltaSharingClient::DeltaSharingClient(std::string filename, std::optional cacheLocation) : restClient(filename) + { + auto path = std::filesystem::current_path().generic_string(); + std::cerr << "Current path: " << path << std::endl; + path.append("/cache"); + this->cacheLocation = cacheLocation.value_or(path); + if (std::filesystem::exists(this->cacheLocation) == false) + std::filesystem::create_directories(this->cacheLocation); + + if (std::filesystem::exists(this->cacheLocation) && std::filesystem::is_directory(this->cacheLocation)) + { + auto p = std::filesystem::status(this->cacheLocation).permissions(); + std::cerr << "Cache directory:" << this->cacheLocation << " Permission: " << ((p & std::filesystem::perms::owner_read) != std::filesystem::perms::none ? "r" : "-") + << ((p & std::filesystem::perms::owner_write) != std::filesystem::perms::none ? "w" : "-") + << ((p & std::filesystem::perms::owner_exec) != std::filesystem::perms::none ? "x" : "-") + << ((p & std::filesystem::perms::group_read) != std::filesystem::perms::none ? "r" : "-") + << ((p & std::filesystem::perms::group_write) != std::filesystem::perms::none ? "w" : "-") + << ((p & std::filesystem::perms::group_exec) != std::filesystem::perms::none ? "x" : "-") + << ((p & std::filesystem::perms::others_read) != std::filesystem::perms::none ? "r" : "-") + << ((p & std::filesystem::perms::others_write) != std::filesystem::perms::none ? "w" : "-") + << ((p & std::filesystem::perms::others_exec) != std::filesystem::perms::none ? "x" : "-") + << '\n'; + } + this->maxThreads = std::thread::hardware_concurrency(); + }; + + std::shared_ptr DeltaSharingClient::LoadAsArrowTable(std::string &url) + { + + if (url.length() == 0) + return std::shared_ptr(); + + int protocolLength = 0; + if ((url.find("http://")) != std::string::npos) + { + protocolLength = 7; + } + + if ((url.find("https://")) != std::string::npos) + { + protocolLength = 8; + } + auto pos = url.find_first_of('?', protocolLength); + auto path = url.substr(protocolLength, pos - protocolLength); // Removing "https://" + + std::vector urlparts; + while ((pos = path.find("/")) != std::string::npos) + { + urlparts.push_back(path.substr(0, pos)); + path.erase(0, pos + 1); + } + if (urlparts.size() != 3) + { + std::cerr << "Invalid URL:" << url << std::endl; + return std::shared_ptr(); + } + std::string tbl = urlparts.back(); + urlparts.pop_back(); + std::string schema = urlparts.back(); + urlparts.pop_back(); + std::string share = urlparts.back(); + + auto completePath = this->cacheLocation + "/" + share + "/" + schema + "/" + tbl; + std::shared_ptr infile; + try + { + PARQUET_ASSIGN_OR_THROW(infile, + arrow::io::ReadableFile::Open(completePath + "/" + path)); + } + catch (parquet::ParquetStatusException &e) + { + std::cerr << "error code:(" << e.status() << ") Message: " << e.what() << std::endl; + return std::shared_ptr(); + } + +/* auto reader_result = parquet::arrow::OpenFile(infile, arrow::default_memory_pool()); + if (!reader_result.ok()) { + throw std::runtime_error(reader_result.status().ToString()); + } + std::unique_ptr reader = std::move(reader_result).ValueOrDie(); + std::shared_ptr table; + + PARQUET_THROW_NOT_OK(reader->ReadTable(&table)); */ + + std::unique_ptr reader; + PARQUET_THROW_NOT_OK( + parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader)); + std::shared_ptr table; + PARQUET_THROW_NOT_OK(reader->ReadTable(&table)); + + return table; + }; + + std::shared_ptr DeltaSharingClient::ReadTableFromCache(std::string &completePath) + { + // To be tested on tables downloaded from cloud + + /* fs::FileSelector selector; + selector.base_dir = completePath; + + + auto filesystem = fs::FileSystemFromUriOrPath(completePath); + auto format = std::make_shared(); + auto format = std::make_shared(); + ds::FileSystemFactoryOptions opts; + + ds::FileSystemDatasetFactory + ds::FileSystemDatasetFactory f; + + auto factor = ds::FileSystemDatasetFactory::Make(filesystem, selector, NULL, NULL); */ + return std::shared_ptr(); + }; + + const std::shared_ptr> DeltaSharingClient::ListShares(int maxResult, std::string pageToken) const + { + return this->restClient.ListShares(maxResult, pageToken); + }; + + const std::shared_ptr> DeltaSharingClient::ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const + { + return this->restClient.ListSchemas(share, maxResult, pageToken); + }; + + const std::shared_ptr> DeltaSharingClient::ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const + { + return this->restClient.ListTables(schema, maxResult, pageToken); + }; + + const std::shared_ptr> DeltaSharingClient::ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const + { + return this->restClient.ListAllTables(share, maxResult, pageToken); + }; + + const std::shared_ptr> DeltaSharingClient::ListFilesInTable(const DeltaSharingProtocol::Table table) const + { + return this->restClient.ListFilesInTable(table); + }; + + const DeltaSharingProtocol::Metadata DeltaSharingClient::QueryTableMetadata(const DeltaSharingProtocol::Table &table) const + { + return this->restClient.QueryTableMetadata(table); + }; +}; diff --git a/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp b/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp new file mode 100644 index 000000000..6f008d882 --- /dev/null +++ b/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp @@ -0,0 +1,272 @@ +/* Node type: delta_sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DeltaSharing +{ + + const std::string DeltaSharingRestClient::user_agent = "delta-sharing-CPP/0.0.1"; + + DeltaSharingRestClient::DeltaSharingRestClient(std::string filename) + { + json j = ReadFromFile(filename); + if(j.empty()) { + return; + } + this->profile = j; + RestClient::init(); + }; + + DeltaSharingRestClient::~DeltaSharingRestClient() + { + std::cerr << "DeltaSharingRestClient destructed" << std::endl; + }; + + const std::shared_ptr> DeltaSharingRestClient::ListShares(int maxResult, std::string pageToken) const + { + std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + RestClient::Response r = c->get("/shares"); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> p; + p = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) + { + DeltaSharingProtocol::Share s = it.value(); + p->push_back(s); + } + return p; + }; + + const std::shared_ptr> DeltaSharingRestClient::ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const + { + std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + share.name + "/schemas"; + RestClient::Response r = c->get(path); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> p; + p = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) + { + DeltaSharingProtocol::Schema s = it.value().get(); + p->push_back(s); + } + return p; + }; + + const std::shared_ptr> DeltaSharingRestClient::ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const + { + std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + schema.share + "/schemas/" + schema.name + "/tables"; + RestClient::Response r = c->get(path); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> t; + t = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) + { + DeltaSharingProtocol::Table s = it.value(); + t->push_back(s); + } + return t; + }; + + const std::shared_ptr> DeltaSharingRestClient::ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const + { + std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + share.name + "/all-tables"; + RestClient::Response r = c->get(path); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> t; + t = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) + { + DeltaSharingProtocol::Table s = it.value(); + t->push_back(s); + } + return t; + }; + + const DeltaSharingProtocol::Metadata DeltaSharingRestClient::QueryTableMetadata(const DeltaSharingProtocol::Table &table) const { + std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + table.share + "/schemas/" + table.schema + "/tables/" + table.name + "/metadata"; + RestClient::Response r = c->get(path); + std::istringstream input; + input.str(r.body); + json j; + DeltaSharingProtocol::Metadata m; + int cnt = 0; + for (std::string line; std::getline(input, line); cnt++) + { + if (cnt == 1) + { + j = json::parse(line); + m = j["metaData"]; + } + } + + return m; + }; + + json DeltaSharingRestClient::ReadFromFile(std::string filename) + { + std::ifstream is; + try { + is.open(filename, std::ifstream::in); + } + catch (std::exception *e) { + json r = {}; + return r; + } + + json j; + is >> j; + is.close(); + return j; + }; + + const DeltaSharingProtocol::DeltaSharingProfile &DeltaSharingRestClient::GetProfile() const + { + return this->profile; + } + + void DeltaSharingRestClient::PopulateCache(std::string url, std::string cacheLocation) { + int protocolLength = 0; + if ((url.find("http://")) != std::string::npos) + { + protocolLength = 7; + } + + if ((url.find("https://")) != std::string::npos) + { + protocolLength = 8; + } + auto pos = url.find_first_of('?', protocolLength); + auto path = url.substr(protocolLength, pos - protocolLength); // Removing "https://" + + std::vector urlparts; + while ((pos = path.find("/")) != std::string::npos) + { + urlparts.push_back(path.substr(0, pos)); + path.erase(0, pos + 1); + } + if (urlparts.size() != 3) + { + std::cerr << "Invalid URL:" << url << std::endl; + return; + } + std::string tbl = urlparts.back(); + urlparts.pop_back(); + std::string schema = urlparts.back(); + urlparts.pop_back(); + std::string share = urlparts.back(); + + auto completePath = cacheLocation + "/" + share + "/" + schema + "/" + tbl; + + if(!std::filesystem::exists(completePath + "/" + path)) + { + std::cerr << completePath+ "/" + path << " does not exist in cache" << std::endl; + std::filesystem::create_directories(completePath); + auto r = this->get(url); + int cnt = 0; + std::cerr << url << " code: " << r.code << std::endl; + + while (this->shouldRetry(r)) + { + cnt++; + std::this_thread::sleep_for(std::chrono::seconds(1)); + if (cnt > 4) + { + std::cerr << "Failed to retrieve file using url: ( Response code: " << r.code << ") Message: " << r.body << std::endl; + return; + } + r = this->get(url); + } + + if (r.code != 200) + { + std::cerr << "Could not read file: " << r.code << " Message: " << r.body << std::endl; + return; + } + + std::fstream f; + f.open(completePath + "/" + path, std::ios::trunc | std::ios::out | std::ios::binary); + f.write(r.body.c_str(), r.body.size()); + f.flush(); + f.close(); + } + }; + + const std::shared_ptr> DeltaSharingRestClient::ListFilesInTable(DeltaSharingProtocol::Table table) const + { + std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + table.share + "/schemas/" + table.schema + "/tables/" + table.name + "/query"; + RestClient::HeaderFields h; + h.insert({"Content-Type", "application/json; charset=UTF-8"}); + h.insert({"Authorization", "Bearer: " + this->profile.bearerToken}); + c->SetHeaders(h); + DeltaSharingProtocol::data d; + json j = d; + RestClient::Response r = c->post(path, j.dump()); + int cnt = 0; + std::istringstream input; + input.str(r.body); + std::shared_ptr> t; + t = std::make_shared>(); + for (std::string line; std::getline(input, line); cnt++) + { + if (cnt > 1) + { + json jf = json::parse(line); + json jf2 = jf["file"]; + DeltaSharingProtocol::File f = jf2; + t->push_back(f); + } + } + + return t; + }; + RestClient::Response DeltaSharingRestClient::get(std::string url) + { + RestClient::Response r = RestClient::get(url); + return r; + }; + + + const bool DeltaSharingRestClient::shouldRetry(RestClient::Response &r) const { + if(r.code == 200) + return false; + if (r.code == 429) { + std::cerr << "Retry operation due to status code: 429" << std::endl; + return true; + } else if (r.code >= 500 && r.code < 600 ) { + std::cerr << "Retry operation due to status code: " << r.code << std::endl; + return true; + } else + return false; + + }; + +}; diff --git a/lib/nodes/delta_sharing/functions.cpp b/lib/nodes/delta_sharing/functions.cpp new file mode 100644 index 000000000..cf5e9fc1b --- /dev/null +++ b/lib/nodes/delta_sharing/functions.cpp @@ -0,0 +1,87 @@ +/* Node type: delta_sharing. + * + * Author: Ritesh Karki + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + + +namespace DeltaSharing +{ + + const std::vector ParseURL(std::string path) + { + std::vector urlparts; + std::string url = path; + auto pos = url.find_last_of('#'); + + if (pos == std::string::npos) + { + std::cerr << "Invalid path: " << url << std::endl; + return std::vector(); + } + urlparts.push_back(url.substr(0, pos)); + url.erase(0, pos + 1); + while ((pos = url.find(".")) != std::string::npos) + { + urlparts.push_back(url.substr(0, pos)); + url.erase(0, pos + 1); + } + urlparts.push_back(url); + if (urlparts.size() != 4) + { + std::cerr << "Path does not follow pattern: #.., " << path << std::endl; + } + return urlparts; + }; + + std::shared_ptr NewDeltaSharingClient(std::string profile, std::optional cacheLocation) + { + return std::make_shared(profile, cacheLocation); + }; + + const std::shared_ptr LoadAsArrowTable(std::string path, int fileno) { + + auto p = ParseURL(path); + if(p.size() != 4) { + std::cerr << "PATH NOT CORRECT: " << path << std::endl; + return std::shared_ptr(); + } + auto cl = NewDeltaSharingClient(p.at(0),std::nullopt); + DeltaSharingProtocol::Table t; + t.name = p.at(3); + t.schema = p.at(2); + t.share = p.at(1); + + auto flist = cl->ListFilesInTable(t); + std::vector writethreads; + + for(long unsigned int i = 0; i < flist->size(); i++) + { + auto arg = flist->at(i).url; + std::thread th(&DeltaSharingClient::PopulateCache, cl, arg); + writethreads.push_back(std::move(th)); + } + + for (auto i = writethreads.begin(); i != writethreads.end(); i++) { + if(i->joinable()) { + i->join(); + } + } + + + if(flist->size() > static_cast(fileno)) { + auto f = flist->at(fileno); + std::cerr << "Number of threads supported: " << cl->GetNumberOfThreads() << std::endl; + + return cl->LoadAsArrowTable(f.url); + } else + return std::shared_ptr(); + }; + + +}; diff --git a/packaging/deps.sh b/packaging/deps.sh index b1b65db09..647f6c97d 100644 --- a/packaging/deps.sh +++ b/packaging/deps.sh @@ -602,6 +602,56 @@ if ! cmake --find-package -DNAME=ghc_filesystem -DCOMPILER_ID=GNU -DLANGUAGE=CXX popd fi +# Build and install Apache Arrow with Parquet and Snappy +if ! cmake --find-package -DNAME=Arrow -DCOMPILER_ID=GNU -DLANGUAGE-CXX -DMODE=EXIST >/dev/null 2>/dev/null && \ + should_build "apache-arrow" "for Arrow/Parquet support"; then + ARROW_TAG=${ARROW_TAG:-apache-arrow-16.1.0} + ARROW_REPO=${ARROW_REPO:-https://github.com/apache/arrow.git} + + git clone ${GIT_OPTS} --branch ${ARROW_TAG} ${ARROW_REPO} apache-arrow + mkdir -p apache-arrow/cpp/build + pushd apache-arrow/cpp/build + + cmake -S ../ \ + -B . \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DARROW_BUILD_SHARED=ON \ + -DARROW_BUILD_STATIC=OFF \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_FILESYSTEM=ON \ + -DARROW_CSV=OFF \ + -DARROW_JSON=ON \ + -DARROW_DATASET=ON \ + -DARROW_PARQUET=ON \ + -DPARQUET_BUILD_EXECUTABLES=OFF \ + -DPARQUET_BUILD_EXAMPLES=OFF + make ${MAKE_OPTS} install + popd +fi + +# Build and install restclient-cpp required for delta_sharing node +if ! find ${PREFIX}/{lib,lib64} -name "*restclient-cpp*" 2>/dev/null | grep -q . && \ + should_build "restclient-cpp" "for the delta-sharing node-type"; then + git clone ${GIT_OPTS} --branch 0.5.2 https://github.com/mrtazz/restclient-cpp.git + pushd restclient-cpp + ./autogen.sh + ./configure ${CONFIGURE_OPTS} + make ${MAKE_OPTS} install + popd +fi + +# Build and install nlohmann/josn required for delta_sharing node +if ! find ${PREFIX}/{include,share} -name "*nlohmann*" 2>/dev/null | grep -q . && \ + should_build "nlohman_json" "for the delta-sharing node-type"; then + git clone https://github.com/nlohmann/json.git json + mkdir -p json/build + pushd json/build + cmake .. + make ${MAKE_OPTS} install + popd +fi + popd >/dev/null # Update linker cache From 837a3150b30215995d319ce24b54611085c98dc4 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Mon, 8 Dec 2025 10:49:02 +0100 Subject: [PATCH 02/14] fix formatting in node delta-sharing Signed-off-by: Ritesh.K --- CMakeLists.txt | 2 +- .../nodes/delta_sharing/delta_sharing.hpp | 18 +- .../delta_sharing/delta_sharing_client.hpp | 59 ++- .../delta_sharing_rest_client.hpp | 65 ++- .../villas/nodes/delta_sharing/functions.hpp | 10 +- .../villas/nodes/delta_sharing/protocol.hpp | 357 ++++++------- lib/nodes/delta_sharing/delta_sharing.cpp | 87 ++-- .../delta_sharing/delta_sharing_client.cpp | 292 ++++++----- .../delta_sharing_rest_client.cpp | 478 +++++++++--------- lib/nodes/delta_sharing/functions.cpp | 143 +++--- 10 files changed, 772 insertions(+), 739 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b3b21ffd..56e334cb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,6 @@ pkg_check_modules(LIBUSB IMPORTED_TARGET libusb-1.0>=1.0.23) pkg_check_modules(NANOMSG IMPORTED_TARGET nanomsg) pkg_check_modules(ARROW IMPORTED_TARGET REQUIRED arrow) pkg_check_modules(PARQUET IMPORTED_TARGET REQUIRED parquet) -# pkg_check_modules(RESTCLIENT IMPORTED_TARGET REQUIRED restclient-cpp) if(NOT NANOMSG_FOUND) pkg_check_modules(NANOMSG IMPORTED_TARGET libnanomsg>=1.0.0) @@ -329,6 +328,7 @@ add_feature_info(NODE_ULDAQ WITH_NODE_ULDAQ "Build with add_feature_info(NODE_WEBRTC WITH_NODE_WEBRTC "Build with webrtc node-type") add_feature_info(NODE_WEBSOCKET WITH_NODE_WEBSOCKET "Build with websocket node-type") add_feature_info(NODE_ZEROMQ WITH_NODE_ZEROMQ "Build with zeromq node-type") +add_feature_info(NODE_DELTASHARING WITH_NODE_DELTASHARING "Build with delta-sharing node-type") if(TOPLEVEL_PROJECT) feature_summary(WHAT ALL VAR FEATURES) diff --git a/include/villas/nodes/delta_sharing/delta_sharing.hpp b/include/villas/nodes/delta_sharing/delta_sharing.hpp index d2a424210..49e1690e2 100644 --- a/include/villas/nodes/delta_sharing/delta_sharing.hpp +++ b/include/villas/nodes/delta_sharing/delta_sharing.hpp @@ -14,10 +14,12 @@ #include -#include #include +#include -namespace arrow { class Table; } +namespace arrow { +class Table; +} namespace villas { namespace node { @@ -37,10 +39,13 @@ struct delta_sharing { // Client and state std::shared_ptr client; - std::shared_ptr> schemas; + std::shared_ptr> + schemas; std::shared_ptr table_ptr; - std::shared_ptr> tables; - std::shared_ptr> shares; + std::shared_ptr> + tables; + std::shared_ptr> + shares; enum class TableOp { TABLE_NOOP, TABLE_READ, TABLE_WRITE } table_op; }; @@ -61,7 +66,8 @@ int deltaSharing_poll_fds(NodeCompat *n, int fds[]); int deltaSharing_read(NodeCompat *n, struct Sample *const smps[], unsigned cnt); -int deltaSharing_write(NodeCompat *n, struct Sample *const smps[], unsigned cnt); +int deltaSharing_write(NodeCompat *n, struct Sample *const smps[], + unsigned cnt); } // namespace node } // namespace villas diff --git a/include/villas/nodes/delta_sharing/delta_sharing_client.hpp b/include/villas/nodes/delta_sharing/delta_sharing_client.hpp index 1eeff28ad..9fda79b91 100644 --- a/include/villas/nodes/delta_sharing/delta_sharing_client.hpp +++ b/include/villas/nodes/delta_sharing/delta_sharing_client.hpp @@ -9,30 +9,43 @@ #include #include -#include + #include -namespace DeltaSharing -{ +#include + +namespace DeltaSharing { + +struct DeltaSharingClient { +public: + DeltaSharingClient(std::string filename, + std::optional cacheLocation); + std::shared_ptr LoadAsArrowTable(std::string &url); + std::shared_ptr ReadTableFromCache(std::string &url); + const std::shared_ptr> + ListShares(int maxResult, std::string pageToken) const; + const std::shared_ptr> + ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, + std::string pageToken) const; + const std::shared_ptr> + ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, + std::string pageToken) const; + const std::shared_ptr> + ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, + std::string pageToken) const; + const std::shared_ptr> + ListFilesInTable(const DeltaSharingProtocol::Table) const; + const DeltaSharingProtocol::Metadata + QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; + const int GetNumberOfThreads() { return this->maxThreads; }; + void PopulateCache(std::string url) { + this->restClient.PopulateCache(url, this->cacheLocation); + }; - struct DeltaSharingClient - { - public: - DeltaSharingClient(std::string filename, std::optional cacheLocation); - std::shared_ptr LoadAsArrowTable(std::string &url); - std::shared_ptr ReadTableFromCache(std::string &url); - const std::shared_ptr> ListShares(int maxResult, std::string pageToken) const; - const std::shared_ptr> ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; - const std::shared_ptr> ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const; - const std::shared_ptr> ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; - const std::shared_ptr> ListFilesInTable(const DeltaSharingProtocol::Table) const; - const DeltaSharingProtocol::Metadata QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; - const int GetNumberOfThreads() { return this->maxThreads; }; - void PopulateCache(std::string url) { this->restClient.PopulateCache(url,this->cacheLocation); }; - protected: - private: - DeltaSharingRestClient restClient; - std::string cacheLocation; - int maxThreads; - }; +protected: +private: + DeltaSharingRestClient restClient; + std::string cacheLocation; + int maxThreads; }; +}; // namespace DeltaSharing diff --git a/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp b/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp index 2045d6999..ee0f6dbeb 100644 --- a/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp +++ b/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp @@ -8,37 +8,46 @@ #pragma once #include -#include #include -#include -#include + +#include #include +#include + +#include using json = nlohmann::json; -namespace DeltaSharing -{ - struct DeltaSharingRestClient - { - public: - DeltaSharingRestClient(std::string filename); - ~DeltaSharingRestClient(); - const std::shared_ptr> ListShares(int maxResult, std::string pageToken) const; - const std::shared_ptr> ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; - const std::shared_ptr> ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const; - const std::shared_ptr> ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const; - const std::shared_ptr> ListFilesInTable(const DeltaSharingProtocol::Table) const; - const DeltaSharingProtocol::Metadata QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; - const DeltaSharingProtocol::DeltaSharingProfile &GetProfile() const; - RestClient::Response get(std::string url); - void PopulateCache(std::string url, std::string cacheLocation); - const bool shouldRetry(RestClient::Response &response) const; - - protected: - json ReadFromFile(std::string filename); - - private: - DeltaSharingProtocol::DeltaSharingProfile profile; - static const std::string user_agent; - }; +namespace DeltaSharing { +struct DeltaSharingRestClient { +public: + DeltaSharingRestClient(std::string filename); + ~DeltaSharingRestClient(); + const std::shared_ptr> + ListShares(int maxResult, std::string pageToken) const; + const std::shared_ptr> + ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, + std::string pageToken) const; + const std::shared_ptr> + ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, + std::string pageToken) const; + const std::shared_ptr> + ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, + std::string pageToken) const; + const std::shared_ptr> + ListFilesInTable(const DeltaSharingProtocol::Table) const; + const DeltaSharingProtocol::Metadata + QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; + const DeltaSharingProtocol::DeltaSharingProfile &GetProfile() const; + RestClient::Response get(std::string url); + void PopulateCache(std::string url, std::string cacheLocation); + const bool shouldRetry(RestClient::Response &response) const; + +protected: + json ReadFromFile(std::string filename); + +private: + DeltaSharingProtocol::DeltaSharingProfile profile; + static const std::string user_agent; }; +}; // namespace DeltaSharing diff --git a/include/villas/nodes/delta_sharing/functions.hpp b/include/villas/nodes/delta_sharing/functions.hpp index 3d1cd0d30..0ae89caa5 100644 --- a/include/villas/nodes/delta_sharing/functions.hpp +++ b/include/villas/nodes/delta_sharing/functions.hpp @@ -7,12 +7,14 @@ #pragma once -#include -#include #include +#include +#include +#include + #include + #include -#include namespace DeltaSharing { @@ -22,6 +24,6 @@ NewDeltaSharingClient(std::string profile, std::optional cacheLocation); const std::shared_ptr LoadAsArrowTable(std::string path, int fileno); -}; +}; // namespace DeltaSharing // namespace DeltaSharing diff --git a/include/villas/nodes/delta_sharing/protocol.hpp b/include/villas/nodes/delta_sharing/protocol.hpp index 1814431ac..ab947275d 100644 --- a/include/villas/nodes/delta_sharing/protocol.hpp +++ b/include/villas/nodes/delta_sharing/protocol.hpp @@ -8,205 +8,170 @@ #pragma once #include -#include #include #include #include #include -namespace DeltaSharing -{ - - namespace DeltaSharingProtocol - { - - using json = nlohmann::json; - - struct DeltaSharingProfile - { - public: - int shareCredentialsVersion; - std::string endpoint; - std::string bearerToken; - std::optional expirationTime; - }; - - inline void from_json(const json &j, DeltaSharingProfile &p) - { - if (j.contains("shareCredentialsVersion")) - { - p.shareCredentialsVersion = j["shareCredentialsVersion"]; - } - if (j.contains("endpoint")) - { - p.endpoint = j["endpoint"]; - } - if (j.contains("bearerToken")) - { - p.bearerToken = j["bearerToken"]; - } - if (j.contains("expirationTime")) - { - p.expirationTime = j["expirationTime"]; - } - }; - - struct Share - { - public: - std::string name = ""; - std::optional id; - }; - - inline void from_json(const json &j, Share &s) - { - s.name = j["name"]; - if (j.contains("id") == false) - { - s.id = std::nullopt; - } - else - { - s.id = j["id"]; - } - }; - - struct Schema - { - public: - std::string name; - std::string share; - }; - - inline void from_json(const json &j, Schema &s) - { - s.name = j["name"]; - if (j.contains("share") == true) - { - s.share = j["share"]; - } - }; - - struct Table - { - public: - std::string name; - std::string share; - std::string schema; - }; - - inline void from_json(const json &j, Table &t) - { - if (j.contains("name")) - { - t.name = j["name"]; - } - if (j.contains("share")) - { - t.share = j["share"]; - } - if (j.contains("schema")) - { - t.schema = j["schema"]; - } - }; - - struct File - { - public: - std::string url; - std::optional id; - std::map partitionValues; - std::size_t size; - std::string stats; - std::optional timestamp; - std::optional version; - }; - - inline void from_json(const json &j, File &f) - { - if (j.contains("url")) - { - f.url = j["url"]; - } - if (j.contains("id")) - { - f.id = j["id"]; - } - if (j.contains("partitionValues")) - { - json arr = j["partitionValues"]; - auto f2 = f.partitionValues; - if (arr.is_array()) - { - for (auto it = arr.begin(); it < arr.end(); it++) - { - f2.insert({it.key(), it.value()}); - } - } - } - if (j.contains("size")) - { - f.size = j["size"]; - } - if (j.contains("stats")) - { - f.stats = j["stats"]; - } - if (j.contains("timestamp")) - { - f.timestamp = j["timestamp"]; - } - if (j.contains("version")) - { - f.version = j["version"]; - } - }; - - struct Format { - std::string provider; - - }; - - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Format, provider) - - struct Metadata - { - Format format; - std::string id; - std::vector partitionColumns; - std::string schemaString; - }; - - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Metadata,format, id, partitionColumns,schemaString) - - struct data - { - std::vector predicateHints; - int limitHint; - }; - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(data, predicateHints, limitHint) - - struct format - { - std::string provider; - }; - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(format, provider) - - struct protocol - { - int minReaderVersion; - }; - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(protocol,minReaderVersion) - - struct stats - { - long long numRecords; - long minValues; - long maxValues; - long nullCount; - }; - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(stats, numRecords, minValues, maxValues, nullCount) - - }; +#include + +namespace DeltaSharing { + +namespace DeltaSharingProtocol { + +using json = nlohmann::json; + +struct DeltaSharingProfile { +public: + int shareCredentialsVersion; + std::string endpoint; + std::string bearerToken; + std::optional expirationTime; +}; + +inline void from_json(const json &j, DeltaSharingProfile &p) { + if (j.contains("shareCredentialsVersion")) { + p.shareCredentialsVersion = j["shareCredentialsVersion"]; + } + if (j.contains("endpoint")) { + p.endpoint = j["endpoint"]; + } + if (j.contains("bearerToken")) { + p.bearerToken = j["bearerToken"]; + } + if (j.contains("expirationTime")) { + p.expirationTime = j["expirationTime"]; + } +}; + +struct Share { +public: + std::string name = ""; + std::optional id; +}; + +inline void from_json(const json &j, Share &s) { + s.name = j["name"]; + if (j.contains("id") == false) { + s.id = std::nullopt; + } else { + s.id = j["id"]; + } +}; + +struct Schema { +public: + std::string name; + std::string share; +}; + +inline void from_json(const json &j, Schema &s) { + s.name = j["name"]; + if (j.contains("share") == true) { + s.share = j["share"]; + } +}; + +struct Table { +public: + std::string name; + std::string share; + std::string schema; +}; + +inline void from_json(const json &j, Table &t) { + if (j.contains("name")) { + t.name = j["name"]; + } + if (j.contains("share")) { + t.share = j["share"]; + } + if (j.contains("schema")) { + t.schema = j["schema"]; + } }; + +struct File { +public: + std::string url; + std::optional id; + std::map partitionValues; + std::size_t size; + std::string stats; + std::optional timestamp; + std::optional version; +}; + +inline void from_json(const json &j, File &f) { + if (j.contains("url")) { + f.url = j["url"]; + } + if (j.contains("id")) { + f.id = j["id"]; + } + if (j.contains("partitionValues")) { + json arr = j["partitionValues"]; + auto f2 = f.partitionValues; + if (arr.is_array()) { + for (auto it = arr.begin(); it < arr.end(); it++) { + f2.insert({it.key(), it.value()}); + } + } + } + if (j.contains("size")) { + f.size = j["size"]; + } + if (j.contains("stats")) { + f.stats = j["stats"]; + } + if (j.contains("timestamp")) { + f.timestamp = j["timestamp"]; + } + if (j.contains("version")) { + f.version = j["version"]; + } +}; + +struct Format { + std::string provider; +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Format, provider) + +struct Metadata { + Format format; + std::string id; + std::vector partitionColumns; + std::string schemaString; +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Metadata, format, id, partitionColumns, + schemaString) + +struct data { + std::vector predicateHints; + int limitHint; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(data, predicateHints, limitHint) + +struct format { + std::string provider; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(format, provider) + +struct protocol { + int minReaderVersion; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(protocol, minReaderVersion) + +struct stats { + long long numRecords; + long minValues; + long maxValues; + long nullCount; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(stats, numRecords, minValues, maxValues, + nullCount) + +}; // namespace DeltaSharingProtocol +}; // namespace DeltaSharing diff --git a/lib/nodes/delta_sharing/delta_sharing.cpp b/lib/nodes/delta_sharing/delta_sharing.cpp index 2a2b6f8ea..4f9b211d6 100644 --- a/lib/nodes/delta_sharing/delta_sharing.cpp +++ b/lib/nodes/delta_sharing/delta_sharing.cpp @@ -5,28 +5,28 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include -#include -#include -#include -#include #include -#include #include -#include #include #include - #include -#include -#include -#include -#include #include +#include +#include #include #include #include +#include +#include +#include + +#include +#include +#include +#include +#include +#include using namespace villas; using namespace villas::node; @@ -50,11 +50,12 @@ int villas::node::deltaSharing_parse(NodeCompat *n, json_t *json) { const char *table = nullptr; int batch_size = 0; - ret = json_unpack_ex(json, &err, 0, - "{ s?: s, s?: s, s?: s, s?: s, s?: s, s?: s, s?: s, s?: i }", - "profile_path", &profile_path, "schema", &schema, - "share", &share, "table", &table,"cache_dir", &cache_dir, "table_path", - &table_path, "op", &op, "batch_size", &batch_size); + ret = json_unpack_ex( + json, &err, 0, + "{ s?: s, s?: s, s?: s, s?: s, s?: s, s?: s, s?: s, s?: i }", + "profile_path", &profile_path, "schema", &schema, "share", &share, + "table", &table, "cache_dir", &cache_dir, "table_path", &table_path, "op", + &op, "batch_size", &batch_size); if (ret) throw ConfigError(json, err, "node-config-node-delta_sharing"); @@ -97,12 +98,11 @@ char *villas::node::deltaSharing_print(NodeCompat *n) { (d->table_op == delta_sharing::TableOp::TABLE_READ ? OP_READ : (d->table_op == delta_sharing::TableOp::TABLE_WRITE ? OP_WRITE - : OP_NOOP)); + : OP_NOOP)); return strdup(info.c_str()); } - int villas::node::deltaSharing_start(NodeCompat *n) { auto *d = n->getData(); @@ -112,7 +112,7 @@ int villas::node::deltaSharing_start(NodeCompat *n) { std::optional cache_opt = d->cache_dir.empty() ? std::nullopt - : std::optional(d->cache_dir); + : std::optional(d->cache_dir); d->client = DeltaSharing::NewDeltaSharingClient(d->profile_path, cache_opt); @@ -131,7 +131,6 @@ int villas::node::deltaSharing_start(NodeCompat *n) { d->tables = d->client->ListAllTables(share, 100, ""); //Check if tables are fetched correctly n->logger->info("Table 1 {}", d->tables->at(0).name); - } return 0; @@ -183,7 +182,7 @@ int villas::node::deltaSharing_poll_fds(NodeCompat *n, int fds[]) { } int villas::node::deltaSharing_read(NodeCompat *n, struct Sample *const smps[], - unsigned cnt) { + unsigned cnt) { auto *d = n->getData(); @@ -201,7 +200,8 @@ int villas::node::deltaSharing_read(NodeCompat *n, struct Sample *const smps[], auto path = DeltaSharing::ParseURL(d->table_path); if (path.size() != 4) { - n->logger->error("Invalid table path format. Expected: server#share.schema.table"); + n->logger->error( + "Invalid table path format. Expected: server#share.schema.table"); return -1; } @@ -263,12 +263,14 @@ int villas::node::deltaSharing_read(NodeCompat *n, struct Sample *const smps[], smp->data[col].f = float_array->Value(i); } case arrow::Type::INT64: { - auto int_array = std::static_pointer_cast(first_chunk); + auto int_array = + std::static_pointer_cast(first_chunk); smp->data[col].i = int_array->Value(i); break; } case arrow::Type::INT32: { - auto int_array = std::static_pointer_cast(first_chunk); + auto int_array = + std::static_pointer_cast(first_chunk); smp->data[col].i = int_array->Value(i); break; } @@ -290,12 +292,12 @@ int villas::node::deltaSharing_read(NodeCompat *n, struct Sample *const smps[], } catch (const std::exception &e) { n->logger->error("Error reading from Delta Sharing table: {}", e.what()); return -1; - } } +} //TODO: write table to delta sharing server. Implementation to be tested int villas::node::deltaSharing_write(NodeCompat *n, struct Sample *const smps[], - unsigned cnt) { + unsigned cnt) { auto *d = n->getData(); if (!d->client) { @@ -311,7 +313,8 @@ int villas::node::deltaSharing_write(NodeCompat *n, struct Sample *const smps[], try { auto path_parts = DeltaSharing::ParseURL(d->table_path); if (path_parts.size() != 4) { - n->logger->error("Invalid table path format. Expected: server#share.schema.table"); + n->logger->error( + "Invalid table path format. Expected: server#share.schema.table"); return -1; } @@ -361,23 +364,23 @@ int villas::node::deltaSharing_write(NodeCompat *n, struct Sample *const smps[], break; } case SignalType::INTEGER: { - std::vector values; - for (unsigned i = 0; i < cnt; i++) { - values.push_back(smps[i]->data[col].i); - } - arrow::Int64Builder builder; - PARQUET_THROW_NOT_OK(builder.AppendValues(values)); - PARQUET_THROW_NOT_OK(builder.Finish(&array)); - break; - } - default: - n->logger->warn("Unsupported signal type for column {}", col); - continue; + std::vector values; + for (unsigned i = 0; i < cnt; i++) { + values.push_back(smps[i]->data[col].i); } + arrow::Int64Builder builder; + PARQUET_THROW_NOT_OK(builder.AppendValues(values)); + PARQUET_THROW_NOT_OK(builder.Finish(&array)); + break; + } + default: + n->logger->warn("Unsupported signal type for column {}", col); + continue; + } - arrays.push_back(array); + arrays.push_back(array); } - // Create Arrow schema and table + // Create Arrow schema and table auto schema = std::make_shared(fields); auto table = arrow::Table::Make(schema, arrays); diff --git a/lib/nodes/delta_sharing/delta_sharing_client.cpp b/lib/nodes/delta_sharing/delta_sharing_client.cpp index faa13cc5d..79283f3c8 100644 --- a/lib/nodes/delta_sharing/delta_sharing_client.cpp +++ b/lib/nodes/delta_sharing/delta_sharing_client.cpp @@ -5,102 +5,128 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include #include #include #include -namespace DeltaSharing -{ - DeltaSharingClient::DeltaSharingClient(std::string filename, std::optional cacheLocation) : restClient(filename) - { - auto path = std::filesystem::current_path().generic_string(); - std::cerr << "Current path: " << path << std::endl; - path.append("/cache"); - this->cacheLocation = cacheLocation.value_or(path); - if (std::filesystem::exists(this->cacheLocation) == false) - std::filesystem::create_directories(this->cacheLocation); - - if (std::filesystem::exists(this->cacheLocation) && std::filesystem::is_directory(this->cacheLocation)) - { - auto p = std::filesystem::status(this->cacheLocation).permissions(); - std::cerr << "Cache directory:" << this->cacheLocation << " Permission: " << ((p & std::filesystem::perms::owner_read) != std::filesystem::perms::none ? "r" : "-") - << ((p & std::filesystem::perms::owner_write) != std::filesystem::perms::none ? "w" : "-") - << ((p & std::filesystem::perms::owner_exec) != std::filesystem::perms::none ? "x" : "-") - << ((p & std::filesystem::perms::group_read) != std::filesystem::perms::none ? "r" : "-") - << ((p & std::filesystem::perms::group_write) != std::filesystem::perms::none ? "w" : "-") - << ((p & std::filesystem::perms::group_exec) != std::filesystem::perms::none ? "x" : "-") - << ((p & std::filesystem::perms::others_read) != std::filesystem::perms::none ? "r" : "-") - << ((p & std::filesystem::perms::others_write) != std::filesystem::perms::none ? "w" : "-") - << ((p & std::filesystem::perms::others_exec) != std::filesystem::perms::none ? "x" : "-") - << '\n'; - } - this->maxThreads = std::thread::hardware_concurrency(); - }; - - std::shared_ptr DeltaSharingClient::LoadAsArrowTable(std::string &url) - { - - if (url.length() == 0) - return std::shared_ptr(); +#include +#include +#include +#include +#include +#include +#include +#include +#include - int protocolLength = 0; - if ((url.find("http://")) != std::string::npos) - { - protocolLength = 7; - } +#include - if ((url.find("https://")) != std::string::npos) - { - protocolLength = 8; - } - auto pos = url.find_first_of('?', protocolLength); - auto path = url.substr(protocolLength, pos - protocolLength); // Removing "https://" - - std::vector urlparts; - while ((pos = path.find("/")) != std::string::npos) - { - urlparts.push_back(path.substr(0, pos)); - path.erase(0, pos + 1); - } - if (urlparts.size() != 3) - { - std::cerr << "Invalid URL:" << url << std::endl; - return std::shared_ptr(); - } - std::string tbl = urlparts.back(); - urlparts.pop_back(); - std::string schema = urlparts.back(); - urlparts.pop_back(); - std::string share = urlparts.back(); - - auto completePath = this->cacheLocation + "/" + share + "/" + schema + "/" + tbl; - std::shared_ptr infile; - try - { - PARQUET_ASSIGN_OR_THROW(infile, - arrow::io::ReadableFile::Open(completePath + "/" + path)); - } - catch (parquet::ParquetStatusException &e) - { - std::cerr << "error code:(" << e.status() << ") Message: " << e.what() << std::endl; - return std::shared_ptr(); - } +namespace DeltaSharing { +DeltaSharingClient::DeltaSharingClient(std::string filename, + std::optional cacheLocation) + : restClient(filename) { + auto path = std::filesystem::current_path().generic_string(); + std::cerr << "Current path: " << path << std::endl; + path.append("/cache"); + this->cacheLocation = cacheLocation.value_or(path); + if (std::filesystem::exists(this->cacheLocation) == false) + std::filesystem::create_directories(this->cacheLocation); + + if (std::filesystem::exists(this->cacheLocation) && + std::filesystem::is_directory(this->cacheLocation)) { + auto p = std::filesystem::status(this->cacheLocation).permissions(); + std::cerr << "Cache directory:" << this->cacheLocation << " Permission: " + << ((p & std::filesystem::perms::owner_read) != + std::filesystem::perms::none + ? "r" + : "-") + << ((p & std::filesystem::perms::owner_write) != + std::filesystem::perms::none + ? "w" + : "-") + << ((p & std::filesystem::perms::owner_exec) != + std::filesystem::perms::none + ? "x" + : "-") + << ((p & std::filesystem::perms::group_read) != + std::filesystem::perms::none + ? "r" + : "-") + << ((p & std::filesystem::perms::group_write) != + std::filesystem::perms::none + ? "w" + : "-") + << ((p & std::filesystem::perms::group_exec) != + std::filesystem::perms::none + ? "x" + : "-") + << ((p & std::filesystem::perms::others_read) != + std::filesystem::perms::none + ? "r" + : "-") + << ((p & std::filesystem::perms::others_write) != + std::filesystem::perms::none + ? "w" + : "-") + << ((p & std::filesystem::perms::others_exec) != + std::filesystem::perms::none + ? "x" + : "-") + << '\n'; + } + this->maxThreads = std::thread::hardware_concurrency(); +}; -/* auto reader_result = parquet::arrow::OpenFile(infile, arrow::default_memory_pool()); +std::shared_ptr +DeltaSharingClient::LoadAsArrowTable(std::string &url) { + + if (url.length() == 0) + return std::shared_ptr(); + + int protocolLength = 0; + if ((url.find("http://")) != std::string::npos) { + protocolLength = 7; + } + + if ((url.find("https://")) != std::string::npos) { + protocolLength = 8; + } + auto pos = url.find_first_of('?', protocolLength); + auto path = + url.substr(protocolLength, pos - protocolLength); // Removing "https://" + + std::vector urlparts; + while ((pos = path.find("/")) != std::string::npos) { + urlparts.push_back(path.substr(0, pos)); + path.erase(0, pos + 1); + } + if (urlparts.size() != 3) { + std::cerr << "Invalid URL:" << url << std::endl; + return std::shared_ptr(); + } + std::string tbl = urlparts.back(); + urlparts.pop_back(); + std::string schema = urlparts.back(); + urlparts.pop_back(); + std::string share = urlparts.back(); + + auto completePath = + this->cacheLocation + "/" + share + "/" + schema + "/" + tbl; + std::shared_ptr infile; + try { + PARQUET_ASSIGN_OR_THROW( + infile, arrow::io::ReadableFile::Open(completePath + "/" + path)); + } catch (parquet::ParquetStatusException &e) { + std::cerr << "error code:(" << e.status() << ") Message: " << e.what() + << std::endl; + return std::shared_ptr(); + } + + /* auto reader_result = parquet::arrow::OpenFile(infile, arrow::default_memory_pool()); if (!reader_result.ok()) { throw std::runtime_error(reader_result.status().ToString()); } @@ -109,20 +135,20 @@ namespace DeltaSharing PARQUET_THROW_NOT_OK(reader->ReadTable(&table)); */ - std::unique_ptr reader; - PARQUET_THROW_NOT_OK( - parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader)); - std::shared_ptr table; - PARQUET_THROW_NOT_OK(reader->ReadTable(&table)); + std::unique_ptr reader; + PARQUET_THROW_NOT_OK( + parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader)); + std::shared_ptr table; + PARQUET_THROW_NOT_OK(reader->ReadTable(&table)); - return table; - }; + return table; +}; - std::shared_ptr DeltaSharingClient::ReadTableFromCache(std::string &completePath) - { - // To be tested on tables downloaded from cloud +std::shared_ptr +DeltaSharingClient::ReadTableFromCache(std::string &completePath) { + // To be tested on tables downloaded from cloud - /* fs::FileSelector selector; + /* fs::FileSelector selector; selector.base_dir = completePath; @@ -135,36 +161,40 @@ namespace DeltaSharing ds::FileSystemDatasetFactory f; auto factor = ds::FileSystemDatasetFactory::Make(filesystem, selector, NULL, NULL); */ - return std::shared_ptr(); - }; - - const std::shared_ptr> DeltaSharingClient::ListShares(int maxResult, std::string pageToken) const - { - return this->restClient.ListShares(maxResult, pageToken); - }; - - const std::shared_ptr> DeltaSharingClient::ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const - { - return this->restClient.ListSchemas(share, maxResult, pageToken); - }; - - const std::shared_ptr> DeltaSharingClient::ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const - { - return this->restClient.ListTables(schema, maxResult, pageToken); - }; - - const std::shared_ptr> DeltaSharingClient::ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const - { - return this->restClient.ListAllTables(share, maxResult, pageToken); - }; - - const std::shared_ptr> DeltaSharingClient::ListFilesInTable(const DeltaSharingProtocol::Table table) const - { - return this->restClient.ListFilesInTable(table); - }; - - const DeltaSharingProtocol::Metadata DeltaSharingClient::QueryTableMetadata(const DeltaSharingProtocol::Table &table) const - { - return this->restClient.QueryTableMetadata(table); - }; + return std::shared_ptr(); +}; + +const std::shared_ptr> +DeltaSharingClient::ListShares(int maxResult, std::string pageToken) const { + return this->restClient.ListShares(maxResult, pageToken); +}; + +const std::shared_ptr> +DeltaSharingClient::ListSchemas(const DeltaSharingProtocol::Share &share, + int maxResult, std::string pageToken) const { + return this->restClient.ListSchemas(share, maxResult, pageToken); +}; + +const std::shared_ptr> +DeltaSharingClient::ListTables(const DeltaSharingProtocol::Schema &schema, + int maxResult, std::string pageToken) const { + return this->restClient.ListTables(schema, maxResult, pageToken); +}; + +const std::shared_ptr> +DeltaSharingClient::ListAllTables(const DeltaSharingProtocol::Share &share, + int maxResult, std::string pageToken) const { + return this->restClient.ListAllTables(share, maxResult, pageToken); +}; + +const std::shared_ptr> +DeltaSharingClient::ListFilesInTable( + const DeltaSharingProtocol::Table table) const { + return this->restClient.ListFilesInTable(table); +}; + +const DeltaSharingProtocol::Metadata DeltaSharingClient::QueryTableMetadata( + const DeltaSharingProtocol::Table &table) const { + return this->restClient.QueryTableMetadata(table); }; +}; // namespace DeltaSharing diff --git a/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp b/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp index 6f008d882..79191ddc0 100644 --- a/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp +++ b/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp @@ -5,268 +5,276 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include -#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include #include -namespace DeltaSharing -{ - - const std::string DeltaSharingRestClient::user_agent = "delta-sharing-CPP/0.0.1"; - - DeltaSharingRestClient::DeltaSharingRestClient(std::string filename) - { - json j = ReadFromFile(filename); - if(j.empty()) { - return; - } - this->profile = j; - RestClient::init(); - }; +#include +#include - DeltaSharingRestClient::~DeltaSharingRestClient() - { - std::cerr << "DeltaSharingRestClient destructed" << std::endl; - }; +#include +#include - const std::shared_ptr> DeltaSharingRestClient::ListShares(int maxResult, std::string pageToken) const - { - std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); - c->SetUserAgent(this->user_agent); - RestClient::Response r = c->get("/shares"); - json j = json::parse(r.body); - auto items = j["items"]; - std::shared_ptr> p; - p = std::make_shared>(); - for (auto it = items.begin(); it < items.end(); it++) - { - DeltaSharingProtocol::Share s = it.value(); - p->push_back(s); - } - return p; - }; +namespace DeltaSharing { - const std::shared_ptr> DeltaSharingRestClient::ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const - { - std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); - c->SetUserAgent(this->user_agent); - std::string path = "/shares/" + share.name + "/schemas"; - RestClient::Response r = c->get(path); - json j = json::parse(r.body); - auto items = j["items"]; - std::shared_ptr> p; - p = std::make_shared>(); - for (auto it = items.begin(); it < items.end(); it++) - { - DeltaSharingProtocol::Schema s = it.value().get(); - p->push_back(s); - } - return p; - }; +const std::string DeltaSharingRestClient::user_agent = + "delta-sharing-CPP/0.0.1"; - const std::shared_ptr> DeltaSharingRestClient::ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, std::string pageToken) const - { - std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); - c->SetUserAgent(this->user_agent); - std::string path = "/shares/" + schema.share + "/schemas/" + schema.name + "/tables"; - RestClient::Response r = c->get(path); - json j = json::parse(r.body); - auto items = j["items"]; - std::shared_ptr> t; - t = std::make_shared>(); - for (auto it = items.begin(); it < items.end(); it++) - { - DeltaSharingProtocol::Table s = it.value(); - t->push_back(s); - } - return t; - }; +DeltaSharingRestClient::DeltaSharingRestClient(std::string filename) { + json j = ReadFromFile(filename); + if (j.empty()) { + return; + } + this->profile = j; + RestClient::init(); +}; - const std::shared_ptr> DeltaSharingRestClient::ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, std::string pageToken) const - { - std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); - c->SetUserAgent(this->user_agent); - std::string path = "/shares/" + share.name + "/all-tables"; - RestClient::Response r = c->get(path); - json j = json::parse(r.body); - auto items = j["items"]; - std::shared_ptr> t; - t = std::make_shared>(); - for (auto it = items.begin(); it < items.end(); it++) - { - DeltaSharingProtocol::Table s = it.value(); - t->push_back(s); - } - return t; - }; +DeltaSharingRestClient::~DeltaSharingRestClient() { + std::cerr << "DeltaSharingRestClient destructed" << std::endl; +}; - const DeltaSharingProtocol::Metadata DeltaSharingRestClient::QueryTableMetadata(const DeltaSharingProtocol::Table &table) const { - std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); - c->SetUserAgent(this->user_agent); - std::string path = "/shares/" + table.share + "/schemas/" + table.schema + "/tables/" + table.name + "/metadata"; - RestClient::Response r = c->get(path); - std::istringstream input; - input.str(r.body); - json j; - DeltaSharingProtocol::Metadata m; - int cnt = 0; - for (std::string line; std::getline(input, line); cnt++) - { - if (cnt == 1) - { - j = json::parse(line); - m = j["metaData"]; - } - } +const std::shared_ptr> +DeltaSharingRestClient::ListShares(int maxResult, std::string pageToken) const { + std::unique_ptr c = + std::unique_ptr( + new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + RestClient::Response r = c->get("/shares"); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> p; + p = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) { + DeltaSharingProtocol::Share s = it.value(); + p->push_back(s); + } + return p; +}; - return m; - }; +const std::shared_ptr> +DeltaSharingRestClient::ListSchemas(const DeltaSharingProtocol::Share &share, + int maxResult, + std::string pageToken) const { + std::unique_ptr c = + std::unique_ptr( + new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + share.name + "/schemas"; + RestClient::Response r = c->get(path); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> p; + p = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) { + DeltaSharingProtocol::Schema s = + it.value().get(); + p->push_back(s); + } + return p; +}; - json DeltaSharingRestClient::ReadFromFile(std::string filename) - { - std::ifstream is; - try { - is.open(filename, std::ifstream::in); - } - catch (std::exception *e) { - json r = {}; - return r; - } +const std::shared_ptr> +DeltaSharingRestClient::ListTables(const DeltaSharingProtocol::Schema &schema, + int maxResult, std::string pageToken) const { + std::unique_ptr c = + std::unique_ptr( + new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = + "/shares/" + schema.share + "/schemas/" + schema.name + "/tables"; + RestClient::Response r = c->get(path); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> t; + t = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) { + DeltaSharingProtocol::Table s = it.value(); + t->push_back(s); + } + return t; +}; - json j; - is >> j; - is.close(); - return j; - }; +const std::shared_ptr> +DeltaSharingRestClient::ListAllTables(const DeltaSharingProtocol::Share &share, + int maxResult, + std::string pageToken) const { + std::unique_ptr c = + std::unique_ptr( + new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + share.name + "/all-tables"; + RestClient::Response r = c->get(path); + json j = json::parse(r.body); + auto items = j["items"]; + std::shared_ptr> t; + t = std::make_shared>(); + for (auto it = items.begin(); it < items.end(); it++) { + DeltaSharingProtocol::Table s = it.value(); + t->push_back(s); + } + return t; +}; - const DeltaSharingProtocol::DeltaSharingProfile &DeltaSharingRestClient::GetProfile() const - { - return this->profile; +const DeltaSharingProtocol::Metadata DeltaSharingRestClient::QueryTableMetadata( + const DeltaSharingProtocol::Table &table) const { + std::unique_ptr c = + std::unique_ptr( + new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + table.share + "/schemas/" + table.schema + + "/tables/" + table.name + "/metadata"; + RestClient::Response r = c->get(path); + std::istringstream input; + input.str(r.body); + json j; + DeltaSharingProtocol::Metadata m; + int cnt = 0; + for (std::string line; std::getline(input, line); cnt++) { + if (cnt == 1) { + j = json::parse(line); + m = j["metaData"]; } + } + + return m; +}; - void DeltaSharingRestClient::PopulateCache(std::string url, std::string cacheLocation) { - int protocolLength = 0; - if ((url.find("http://")) != std::string::npos) - { - protocolLength = 7; - } +json DeltaSharingRestClient::ReadFromFile(std::string filename) { + std::ifstream is; + try { + is.open(filename, std::ifstream::in); + } catch (std::exception *e) { + json r = {}; + return r; + } - if ((url.find("https://")) != std::string::npos) - { - protocolLength = 8; - } - auto pos = url.find_first_of('?', protocolLength); - auto path = url.substr(protocolLength, pos - protocolLength); // Removing "https://" + json j; + is >> j; + is.close(); + return j; +}; - std::vector urlparts; - while ((pos = path.find("/")) != std::string::npos) - { - urlparts.push_back(path.substr(0, pos)); - path.erase(0, pos + 1); - } - if (urlparts.size() != 3) - { - std::cerr << "Invalid URL:" << url << std::endl; - return; - } - std::string tbl = urlparts.back(); - urlparts.pop_back(); - std::string schema = urlparts.back(); - urlparts.pop_back(); - std::string share = urlparts.back(); +const DeltaSharingProtocol::DeltaSharingProfile & +DeltaSharingRestClient::GetProfile() const { + return this->profile; +} - auto completePath = cacheLocation + "/" + share + "/" + schema + "/" + tbl; +void DeltaSharingRestClient::PopulateCache(std::string url, + std::string cacheLocation) { + int protocolLength = 0; + if ((url.find("http://")) != std::string::npos) { + protocolLength = 7; + } - if(!std::filesystem::exists(completePath + "/" + path)) - { - std::cerr << completePath+ "/" + path << " does not exist in cache" << std::endl; - std::filesystem::create_directories(completePath); - auto r = this->get(url); - int cnt = 0; - std::cerr << url << " code: " << r.code << std::endl; + if ((url.find("https://")) != std::string::npos) { + protocolLength = 8; + } + auto pos = url.find_first_of('?', protocolLength); + auto path = + url.substr(protocolLength, pos - protocolLength); // Removing "https://" - while (this->shouldRetry(r)) - { - cnt++; - std::this_thread::sleep_for(std::chrono::seconds(1)); - if (cnt > 4) - { - std::cerr << "Failed to retrieve file using url: ( Response code: " << r.code << ") Message: " << r.body << std::endl; - return; - } - r = this->get(url); - } + std::vector urlparts; + while ((pos = path.find("/")) != std::string::npos) { + urlparts.push_back(path.substr(0, pos)); + path.erase(0, pos + 1); + } + if (urlparts.size() != 3) { + std::cerr << "Invalid URL:" << url << std::endl; + return; + } + std::string tbl = urlparts.back(); + urlparts.pop_back(); + std::string schema = urlparts.back(); + urlparts.pop_back(); + std::string share = urlparts.back(); - if (r.code != 200) - { - std::cerr << "Could not read file: " << r.code << " Message: " << r.body << std::endl; - return; - } + auto completePath = cacheLocation + "/" + share + "/" + schema + "/" + tbl; - std::fstream f; - f.open(completePath + "/" + path, std::ios::trunc | std::ios::out | std::ios::binary); - f.write(r.body.c_str(), r.body.size()); - f.flush(); - f.close(); - } - }; + if (!std::filesystem::exists(completePath + "/" + path)) { + std::cerr << completePath + "/" + path << " does not exist in cache" + << std::endl; + std::filesystem::create_directories(completePath); + auto r = this->get(url); + int cnt = 0; + std::cerr << url << " code: " << r.code << std::endl; - const std::shared_ptr> DeltaSharingRestClient::ListFilesInTable(DeltaSharingProtocol::Table table) const - { - std::unique_ptr c = std::unique_ptr(new RestClient::Connection(this->profile.endpoint)); - c->SetUserAgent(this->user_agent); - std::string path = "/shares/" + table.share + "/schemas/" + table.schema + "/tables/" + table.name + "/query"; - RestClient::HeaderFields h; - h.insert({"Content-Type", "application/json; charset=UTF-8"}); - h.insert({"Authorization", "Bearer: " + this->profile.bearerToken}); - c->SetHeaders(h); - DeltaSharingProtocol::data d; - json j = d; - RestClient::Response r = c->post(path, j.dump()); - int cnt = 0; - std::istringstream input; - input.str(r.body); - std::shared_ptr> t; - t = std::make_shared>(); - for (std::string line; std::getline(input, line); cnt++) - { - if (cnt > 1) - { - json jf = json::parse(line); - json jf2 = jf["file"]; - DeltaSharingProtocol::File f = jf2; - t->push_back(f); - } - } + while (this->shouldRetry(r)) { + cnt++; + std::this_thread::sleep_for(std::chrono::seconds(1)); + if (cnt > 4) { + std::cerr << "Failed to retrieve file using url: ( Response code: " + << r.code << ") Message: " << r.body << std::endl; + return; + } + r = this->get(url); + } - return t; - }; - RestClient::Response DeltaSharingRestClient::get(std::string url) - { - RestClient::Response r = RestClient::get(url); - return r; - }; + if (r.code != 200) { + std::cerr << "Could not read file: " << r.code << " Message: " << r.body + << std::endl; + return; + } + std::fstream f; + f.open(completePath + "/" + path, + std::ios::trunc | std::ios::out | std::ios::binary); + f.write(r.body.c_str(), r.body.size()); + f.flush(); + f.close(); + } +}; - const bool DeltaSharingRestClient::shouldRetry(RestClient::Response &r) const { - if(r.code == 200) - return false; - if (r.code == 429) { - std::cerr << "Retry operation due to status code: 429" << std::endl; - return true; - } else if (r.code >= 500 && r.code < 600 ) { - std::cerr << "Retry operation due to status code: " << r.code << std::endl; - return true; - } else - return false; +const std::shared_ptr> +DeltaSharingRestClient::ListFilesInTable( + DeltaSharingProtocol::Table table) const { + std::unique_ptr c = + std::unique_ptr( + new RestClient::Connection(this->profile.endpoint)); + c->SetUserAgent(this->user_agent); + std::string path = "/shares/" + table.share + "/schemas/" + table.schema + + "/tables/" + table.name + "/query"; + RestClient::HeaderFields h; + h.insert({"Content-Type", "application/json; charset=UTF-8"}); + h.insert({"Authorization", "Bearer: " + this->profile.bearerToken}); + c->SetHeaders(h); + DeltaSharingProtocol::data d; + json j = d; + RestClient::Response r = c->post(path, j.dump()); + int cnt = 0; + std::istringstream input; + input.str(r.body); + std::shared_ptr> t; + t = std::make_shared>(); + for (std::string line; std::getline(input, line); cnt++) { + if (cnt > 1) { + json jf = json::parse(line); + json jf2 = jf["file"]; + DeltaSharingProtocol::File f = jf2; + t->push_back(f); + } + } - }; + return t; +}; +RestClient::Response DeltaSharingRestClient::get(std::string url) { + RestClient::Response r = RestClient::get(url); + return r; +}; +const bool DeltaSharingRestClient::shouldRetry(RestClient::Response &r) const { + if (r.code == 200) + return false; + if (r.code == 429) { + std::cerr << "Retry operation due to status code: 429" << std::endl; + return true; + } else if (r.code >= 500 && r.code < 600) { + std::cerr << "Retry operation due to status code: " << r.code << std::endl; + return true; + } else + return false; }; + +}; // namespace DeltaSharing diff --git a/lib/nodes/delta_sharing/functions.cpp b/lib/nodes/delta_sharing/functions.cpp index cf5e9fc1b..2bdc7ebcd 100644 --- a/lib/nodes/delta_sharing/functions.cpp +++ b/lib/nodes/delta_sharing/functions.cpp @@ -7,81 +7,78 @@ #include #include -#include - - -namespace DeltaSharing -{ - - const std::vector ParseURL(std::string path) - { - std::vector urlparts; - std::string url = path; - auto pos = url.find_last_of('#'); - - if (pos == std::string::npos) - { - std::cerr << "Invalid path: " << url << std::endl; - return std::vector(); - } - urlparts.push_back(url.substr(0, pos)); - url.erase(0, pos + 1); - while ((pos = url.find(".")) != std::string::npos) - { - urlparts.push_back(url.substr(0, pos)); - url.erase(0, pos + 1); - } - urlparts.push_back(url); - if (urlparts.size() != 4) - { - std::cerr << "Path does not follow pattern: #..
, " << path << std::endl; - } - return urlparts; - }; - - std::shared_ptr NewDeltaSharingClient(std::string profile, std::optional cacheLocation) - { - return std::make_shared(profile, cacheLocation); - }; - - const std::shared_ptr LoadAsArrowTable(std::string path, int fileno) { - - auto p = ParseURL(path); - if(p.size() != 4) { - std::cerr << "PATH NOT CORRECT: " << path << std::endl; - return std::shared_ptr(); - } - auto cl = NewDeltaSharingClient(p.at(0),std::nullopt); - DeltaSharingProtocol::Table t; - t.name = p.at(3); - t.schema = p.at(2); - t.share = p.at(1); - auto flist = cl->ListFilesInTable(t); - std::vector writethreads; - - for(long unsigned int i = 0; i < flist->size(); i++) - { - auto arg = flist->at(i).url; - std::thread th(&DeltaSharingClient::PopulateCache, cl, arg); - writethreads.push_back(std::move(th)); - } - - for (auto i = writethreads.begin(); i != writethreads.end(); i++) { - if(i->joinable()) { - i->join(); - } - } - - - if(flist->size() > static_cast(fileno)) { - auto f = flist->at(fileno); - std::cerr << "Number of threads supported: " << cl->GetNumberOfThreads() << std::endl; +#include - return cl->LoadAsArrowTable(f.url); - } else - return std::shared_ptr(); - }; +namespace DeltaSharing { + +const std::vector ParseURL(std::string path) { + std::vector urlparts; + std::string url = path; + auto pos = url.find_last_of('#'); + + if (pos == std::string::npos) { + std::cerr << "Invalid path: " << url << std::endl; + return std::vector(); + } + urlparts.push_back(url.substr(0, pos)); + url.erase(0, pos + 1); + while ((pos = url.find(".")) != std::string::npos) { + urlparts.push_back(url.substr(0, pos)); + url.erase(0, pos + 1); + } + urlparts.push_back(url); + if (urlparts.size() != 4) { + std::cerr + << "Path does not follow pattern: #..
, " + << path << std::endl; + } + return urlparts; +}; +std::shared_ptr +NewDeltaSharingClient(std::string profile, + std::optional cacheLocation) { + return std::make_shared(profile, cacheLocation); +}; +const std::shared_ptr LoadAsArrowTable(std::string path, + int fileno) { + + auto p = ParseURL(path); + if (p.size() != 4) { + std::cerr << "PATH NOT CORRECT: " << path << std::endl; + return std::shared_ptr(); + } + auto cl = NewDeltaSharingClient(p.at(0), std::nullopt); + DeltaSharingProtocol::Table t; + t.name = p.at(3); + t.schema = p.at(2); + t.share = p.at(1); + + auto flist = cl->ListFilesInTable(t); + std::vector writethreads; + + for (long unsigned int i = 0; i < flist->size(); i++) { + auto arg = flist->at(i).url; + std::thread th(&DeltaSharingClient::PopulateCache, cl, arg); + writethreads.push_back(std::move(th)); + } + + for (auto i = writethreads.begin(); i != writethreads.end(); i++) { + if (i->joinable()) { + i->join(); + } + } + + if (flist->size() > static_cast(fileno)) { + auto f = flist->at(fileno); + std::cerr << "Number of threads supported: " << cl->GetNumberOfThreads() + << std::endl; + + return cl->LoadAsArrowTable(f.url); + } else + return std::shared_ptr(); }; + +}; // namespace DeltaSharing From e2837669989d7b7f03fe21f771f80a8c937dd58b Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Wed, 10 Dec 2025 17:06:23 +0000 Subject: [PATCH 03/14] fix issue with restclient-cpp linking during build Signed-off-by: Ritesh.K --- CMakeLists.txt | 1 + packaging/deps.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 56e334cb1..504c17428 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,7 @@ pkg_check_modules(LIBUSB IMPORTED_TARGET libusb-1.0>=1.0.23) pkg_check_modules(NANOMSG IMPORTED_TARGET nanomsg) pkg_check_modules(ARROW IMPORTED_TARGET REQUIRED arrow) pkg_check_modules(PARQUET IMPORTED_TARGET REQUIRED parquet) +pkg_check_modules(RESTCLIENTCPP REQUIRED restclient-cpp) if(NOT NANOMSG_FOUND) pkg_check_modules(NANOMSG IMPORTED_TARGET libnanomsg>=1.0.0) diff --git a/packaging/deps.sh b/packaging/deps.sh index 647f6c97d..0d2a4eaca 100644 --- a/packaging/deps.sh +++ b/packaging/deps.sh @@ -637,7 +637,10 @@ if ! find ${PREFIX}/{lib,lib64} -name "*restclient-cpp*" 2>/dev/null | grep -q . pushd restclient-cpp ./autogen.sh ./configure ${CONFIGURE_OPTS} + pc_src=restclient-cpp.pc + sed -i -E 's/^[[:space:]]*Requires[[:space:]]*:[[:space:]]*curl[[:space:]]*$/Requires.private: libcurl/' "$pc_src" make ${MAKE_OPTS} install + cp ./restclient-cpp.pc /usr/local/lib/pkgconfig/ popd fi From 04cc5e3e72be1ea03e06d032bd239263995850f6 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Wed, 10 Dec 2025 23:13:09 +0100 Subject: [PATCH 04/14] fix issue with restclient-cpp linking with libvillas Signed-off-by: Ritesh.K --- lib/nodes/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/nodes/CMakeLists.txt b/lib/nodes/CMakeLists.txt index 1b6a640fa..f6ed30a08 100644 --- a/lib/nodes/CMakeLists.txt +++ b/lib/nodes/CMakeLists.txt @@ -204,10 +204,10 @@ endif() # Enable Delta Sharing if(WITH_NODE_DELTASHARING) - list(APPEND NODE_SRC delta_sharing/delta_sharing.cpp delta_sharing/delta_sharing_client.cpp + list(APPEND NODE_SRC delta_sharing/delta_sharing.cpp delta_sharing/delta_sharing_client.cpp delta_sharing/delta_sharing_rest_client.cpp delta_sharing/functions.cpp ) - list(APPEND LIBRARIES ${ARROW_LIBRARIES} ${PARQUET_LIBRARIES} ${RESTCLIENT_LIBRARIES}) + list(APPEND LIBRARIES ${ARROW_LIBRARIES} ${PARQUET_LIBRARIES} ${RESTCLIENTCPP_LIBRARIES}) endif() add_library(nodes STATIC ${NODE_SRC}) From 9a33fe65731bf3b63e9cbf1731103fb959e755b0 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Thu, 18 Dec 2025 13:40:03 +0100 Subject: [PATCH 05/14] fix issue with arrow packages being checked when flag 'WITH_NODE_DELTASHARING' is not set Signed-off-by: Ritesh.K --- CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 504c17428..5b72fb2a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,9 +115,12 @@ pkg_check_modules(CGRAPH IMPORTED_TARGET libcgraph>=2.30) pkg_check_modules(GVC IMPORTED_TARGET libgvc>=2.30) pkg_check_modules(LIBUSB IMPORTED_TARGET libusb-1.0>=1.0.23) pkg_check_modules(NANOMSG IMPORTED_TARGET nanomsg) -pkg_check_modules(ARROW IMPORTED_TARGET REQUIRED arrow) -pkg_check_modules(PARQUET IMPORTED_TARGET REQUIRED parquet) -pkg_check_modules(RESTCLIENTCPP REQUIRED restclient-cpp) + +if(WITH_NODE_DELTASHARING) + pkg_check_modules(ARROW IMPORTED_TARGET REQUIRED arrow) + pkg_check_modules(PARQUET IMPORTED_TARGET REQUIRED parquet) + pkg_check_modules(RESTCLIENTCPP REQUIRED restclient-cpp) +endif() if(NOT NANOMSG_FOUND) pkg_check_modules(NANOMSG IMPORTED_TARGET libnanomsg>=1.0.0) From e3bf3bec9e59d37bc9998e945193ec1823e65207 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Sat, 20 Dec 2025 15:57:10 +0000 Subject: [PATCH 06/14] fix cmake workflow when missing arrow packages for node delta_sharing Signed-off-by: Ritesh.K --- CMakeLists.txt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b72fb2a0..aef6e7299 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,17 +115,19 @@ pkg_check_modules(CGRAPH IMPORTED_TARGET libcgraph>=2.30) pkg_check_modules(GVC IMPORTED_TARGET libgvc>=2.30) pkg_check_modules(LIBUSB IMPORTED_TARGET libusb-1.0>=1.0.23) pkg_check_modules(NANOMSG IMPORTED_TARGET nanomsg) - -if(WITH_NODE_DELTASHARING) - pkg_check_modules(ARROW IMPORTED_TARGET REQUIRED arrow) - pkg_check_modules(PARQUET IMPORTED_TARGET REQUIRED parquet) - pkg_check_modules(RESTCLIENTCPP REQUIRED restclient-cpp) -endif() +pkg_check_modules(ARROW IMPORTED_TARGET arrow) +pkg_check_modules(PARQUET IMPORTED_TARGET parquet) +pkg_check_modules(RESTCLIENTCPP restclient-cpp) if(NOT NANOMSG_FOUND) pkg_check_modules(NANOMSG IMPORTED_TARGET libnanomsg>=1.0.0) endif() +if(ARROW_FOUND AND PARQUET_FOUND AND RESTCLIENTCPP_FOUND) + set(DELTALIBS_FOUND ON) +else() + set(DELTALIBS_FOUND OFF) +endif() if (REDISPP_FOUND) file(READ "${REDISPP_INCLUDEDIR}/sw/redis++/tls.h" CONTENTS) @@ -187,6 +189,7 @@ cmake_dependent_option(WITH_SRC "Build executables" cmake_dependent_option(WITH_TESTS "Run tests" "${WITH_DEFAULTS}" "TOPLEVEL_PROJECT" OFF) cmake_dependent_option(WITH_TOOLS "Build auxilary tools" "${WITH_DEFAULTS}" "TOPLEVEL_PROJECT" OFF) cmake_dependent_option(WITH_WEB "Build with internal webserver" "${WITH_DEFAULTS}" "LIBWEBSOCKETS_FOUND" OFF) +cmake_dependent_option(WITH_DELTASHARING "Build with delta sharing active" "${WITH_DEFAULTS}" "DELTALIBS_FOUND" OFF) cmake_dependent_option(WITH_NODE_AMQP "Build with amqp node-type" "${WITH_DEFAULTS}" "RABBITMQ_C_FOUND" OFF) cmake_dependent_option(WITH_NODE_CAN "Build with can node-type" "${WITH_DEFAULTS}" "" OFF) @@ -221,7 +224,7 @@ cmake_dependent_option(WITH_NODE_ULDAQ "Build with uldaq node-type" cmake_dependent_option(WITH_NODE_WEBRTC "Build with webrtc node-type" "${WITH_DEFAULTS}" "WITH_WEB; LibDataChannel_FOUND" OFF) cmake_dependent_option(WITH_NODE_WEBSOCKET "Build with websocket node-type" "${WITH_DEFAULTS}" "WITH_WEB" OFF) cmake_dependent_option(WITH_NODE_ZEROMQ "Build with zeromq node-type" "${WITH_DEFAULTS}" "LIBZMQ_FOUND; NOT WITHOUT_GPL" OFF) -cmake_dependent_option(WITH_NODE_DELTASHARING "Build with delta_sharing node-type" "${WITH_DEFAULTS}" "" OFF) +cmake_dependent_option(WITH_NODE_DELTASHARING "Build with delta_sharing node-type" "${WITH_DEFAULTS}" "WITH_DELTASHARING" OFF) # Set a default for the build type if("${CMAKE_BUILD_TYPE}" STREQUAL "") From 902de99cc2ed514a19be4de9221a2a439de068c1 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Sun, 4 Jan 2026 22:03:57 +0100 Subject: [PATCH 07/14] fix (node-delta-sharing): Fix cpp files formatting Signed-off-by: Ritesh.K --- common/include/villas/kernel/devices/device.hpp | 2 +- .../include/villas/kernel/devices/ip_device.hpp | 2 +- .../villas/kernel/devices/linux_driver.hpp | 4 ++-- .../villas/kernel/devices/platform_device.hpp | 4 ++-- common/include/villas/utils.hpp | 8 ++++---- common/lib/kernel/devices/device_connection.cpp | 2 +- fpga/include/villas/fpga/ips/i2c.hpp | 5 ++--- fpga/include/villas/fpga/platform_card.hpp | 2 +- fpga/include/villas/fpga/utils.hpp | 6 +++--- fpga/lib/core_connection.cpp | 2 +- fpga/src/villas-fpga-pipe.cpp | 2 +- include/villas/nodes/iec61850.hpp | 3 +-- include/villas/nodes/test_rtt.hpp | 4 ++-- lib/nodes/can.cpp | 10 +++++----- tests/unit/queue_signalled.cpp | 16 ++++++++-------- 15 files changed, 35 insertions(+), 37 deletions(-) diff --git a/common/include/villas/kernel/devices/device.hpp b/common/include/villas/kernel/devices/device.hpp index d6602da10..a3b1a8656 100644 --- a/common/include/villas/kernel/devices/device.hpp +++ b/common/include/villas/kernel/devices/device.hpp @@ -19,7 +19,7 @@ namespace devices { class Device { public: - virtual ~Device(){}; + virtual ~Device() {}; virtual std::optional> driver() const = 0; virtual std::optional iommu_group() const = 0; diff --git a/common/include/villas/kernel/devices/ip_device.hpp b/common/include/villas/kernel/devices/ip_device.hpp index 4ec192f08..374383e78 100644 --- a/common/include/villas/kernel/devices/ip_device.hpp +++ b/common/include/villas/kernel/devices/ip_device.hpp @@ -25,7 +25,7 @@ class IpDevice : public PlatformDevice { private: IpDevice() = delete; IpDevice(const fs::path valid_path) //! Dont allow unvalidated paths - : PlatformDevice(valid_path){}; + : PlatformDevice(valid_path) {}; public: size_t addr() const; diff --git a/common/include/villas/kernel/devices/linux_driver.hpp b/common/include/villas/kernel/devices/linux_driver.hpp index b1c01cb14..74dadb4f0 100644 --- a/common/include/villas/kernel/devices/linux_driver.hpp +++ b/common/include/villas/kernel/devices/linux_driver.hpp @@ -33,11 +33,11 @@ class LinuxDriver : public Driver { public: LinuxDriver(const fs::path path) : LinuxDriver(path, path / fs::path(BIND_DEFAULT), - path / fs::path(UNBIND_DEFAULT)){}; + path / fs::path(UNBIND_DEFAULT)) {}; LinuxDriver(const fs::path path, const fs::path bind_path, const fs::path unbind_path) - : path(path), bind_path(bind_path), unbind_path(unbind_path){}; + : path(path), bind_path(bind_path), unbind_path(unbind_path) {}; public: void attach(const Device &device) const override; diff --git a/common/include/villas/kernel/devices/platform_device.hpp b/common/include/villas/kernel/devices/platform_device.hpp index 1071a5a84..bad32f2cb 100644 --- a/common/include/villas/kernel/devices/platform_device.hpp +++ b/common/include/villas/kernel/devices/platform_device.hpp @@ -29,12 +29,12 @@ class PlatformDevice : public Device { public: PlatformDevice(const fs::path path) : PlatformDevice(path, fs::path(PROBE_DEFAULT), - path / fs::path(OVERRIDE_DEFAULT)){}; + path / fs::path(OVERRIDE_DEFAULT)) {}; PlatformDevice(const fs::path path, const fs::path probe_path, const fs::path override_path) : m_path(path), m_probe_path(probe_path), - m_override_path(override_path){}; + m_override_path(override_path) {}; // Implement device interface std::optional> driver() const override; diff --git a/common/include/villas/utils.hpp b/common/include/villas/utils.hpp index 1ebb1cf08..0a0ad01e9 100644 --- a/common/include/villas/utils.hpp +++ b/common/include/villas/utils.hpp @@ -52,7 +52,7 @@ #ifdef ALIGN #undef ALIGN #endif -#define ALIGN(x, a) ALIGN_MASK(x, (uintptr_t)(a)-1) +#define ALIGN(x, a) ALIGN_MASK(x, (uintptr_t)(a) - 1) #define ALIGN_MASK(x, m) (((uintptr_t)(x) + (m)) & ~(m)) #define IS_ALIGNED(x, a) (ALIGN(x, a) == (uintptr_t)x) @@ -64,13 +64,13 @@ } while (0) // Round-up integer division -#define CEIL(x, y) (((x) + (y)-1) / (y)) +#define CEIL(x, y) (((x) + (y) - 1) / (y)) // Get nearest up-rounded power of 2 -#define LOG2_CEIL(x) (1 << (villas::utils::log2i((x)-1) + 1)) +#define LOG2_CEIL(x) (1 << (villas::utils::log2i((x) - 1) + 1)) // Check if the number is a power of 2 -#define IS_POW2(x) (((x) != 0) && !((x) & ((x)-1))) +#define IS_POW2(x) (((x) != 0) && !((x) & ((x) - 1))) // Calculate the number of elements in an array. #define ARRAY_LEN(a) (sizeof(a) / sizeof(a)[0]) diff --git a/common/lib/kernel/devices/device_connection.cpp b/common/lib/kernel/devices/device_connection.cpp index b32872f02..28b156e79 100644 --- a/common/lib/kernel/devices/device_connection.cpp +++ b/common/lib/kernel/devices/device_connection.cpp @@ -18,7 +18,7 @@ namespace devices { DeviceConnection::DeviceConnection( std::shared_ptr vfio_device) - : logger(villas::Log::get("DeviceConnection")), vfio_device(vfio_device){}; + : logger(villas::Log::get("DeviceConnection")), vfio_device(vfio_device) {}; DeviceConnection DeviceConnection::from( const villas::kernel::devices::Device &device, diff --git a/fpga/include/villas/fpga/ips/i2c.hpp b/fpga/include/villas/fpga/ips/i2c.hpp index b90f34fac..1854b1cf8 100644 --- a/fpga/include/villas/fpga/ips/i2c.hpp +++ b/fpga/include/villas/fpga/ips/i2c.hpp @@ -20,8 +20,7 @@ namespace fpga { namespace ip { #define I2C_SWTICH_ADDR 0x70 -#define I2C_SWITCH_CHANNEL_MAP \ - { 0x20, 0x80, 0x02, 0x08, 0x10, 0x40, 0x01, 0x04 } +#define I2C_SWITCH_CHANNEL_MAP {0x20, 0x80, 0x02, 0x08, 0x10, 0x40, 0x01, 0x04} #define I2C_IOEXT_ADDR 0x20 #define I2C_IOEXT_REG_DIR 0x03 #define I2C_IOEXT_REG_OUT 0x01 @@ -48,7 +47,7 @@ class I2c : public Node { public: Switch(I2c *i2c, uint8_t address, Logger logger = villas::Log::get("i2c")) : i2c(i2c), address(address), channel(0), readOnce(false), switchLock(), - logger(logger){}; + logger(logger) {}; Switch(const Switch &other) = delete; Switch &operator=(const Switch &other) = delete; void setChannel(uint8_t channel); diff --git a/fpga/include/villas/fpga/platform_card.hpp b/fpga/include/villas/fpga/platform_card.hpp index 22fc4b93b..e122caf0d 100644 --- a/fpga/include/villas/fpga/platform_card.hpp +++ b/fpga/include/villas/fpga/platform_card.hpp @@ -25,7 +25,7 @@ class PlatformCard : public Card { public: PlatformCard(std::shared_ptr vfioContainer); - ~PlatformCard(){}; + ~PlatformCard() {}; std::vector core_connections; diff --git a/fpga/include/villas/fpga/utils.hpp b/fpga/include/villas/fpga/utils.hpp index d8493063c..f7025b990 100644 --- a/fpga/include/villas/fpga/utils.hpp +++ b/fpga/include/villas/fpga/utils.hpp @@ -90,7 +90,7 @@ class BufferedSampleFormatter { BufferedSampleFormatter(const size_t bufSamples, const size_t bufSampleSize) : buf(bufSamples * bufSampleSize + 1), // Leave room for a final `\0' bufSamples(bufSamples), bufSampleSize(bufSampleSize), - currentBufLoc(0){}; + currentBufLoc(0) {}; BufferedSampleFormatter() = delete; BufferedSampleFormatter(const BufferedSampleFormatter &) = delete; virtual char *nextBufPos() { return &buf[(currentBufLoc++) * bufSampleSize]; } @@ -99,7 +99,7 @@ class BufferedSampleFormatter { class BufferedSampleFormatterShort : public BufferedSampleFormatter { public: BufferedSampleFormatterShort(size_t bufSizeInSamples) - : BufferedSampleFormatter(bufSizeInSamples, formatStringSize){}; + : BufferedSampleFormatter(bufSizeInSamples, formatStringSize) {}; virtual void format(float value) override { size_t chars; @@ -120,7 +120,7 @@ class BufferedSampleFormatterLong : public BufferedSampleFormatter { public: BufferedSampleFormatterLong(size_t bufSizeInSamples) : BufferedSampleFormatter(bufSizeInSamples, formatStringSize), - sampleCnt(0){}; + sampleCnt(0) {}; virtual void format(float value) override { if (std::snprintf(nextBufPos(), formatStringSize + 1, formatString, diff --git a/fpga/lib/core_connection.cpp b/fpga/lib/core_connection.cpp index 242009a34..c70d96536 100644 --- a/fpga/lib/core_connection.cpp +++ b/fpga/lib/core_connection.cpp @@ -20,7 +20,7 @@ CoreConnection::CoreConnection( std::shared_ptr ip, villas::kernel::devices::DeviceConnection device_connection) : logger(villas::Log::get("CoreConnection")), ip(ip), - device_connection(device_connection){}; + device_connection(device_connection) {}; CoreConnection CoreConnection::from(std::shared_ptr ip, diff --git a/fpga/src/villas-fpga-pipe.cpp b/fpga/src/villas-fpga-pipe.cpp index 3aba1657a..0b18ba1ae 100644 --- a/fpga/src/villas-fpga-pipe.cpp +++ b/fpga/src/villas-fpga-pipe.cpp @@ -78,7 +78,7 @@ int main(int argc, char *argv[]) { for (auto aurora : aurora_channels) aurora->dump(); - // Configure Crossbar switch + // Configure Crossbar switch #if 1 aurora_channels[3]->connect(aurora_channels[3]->getDefaultMasterPort(), dma->getDefaultSlavePort()); diff --git a/include/villas/nodes/iec61850.hpp b/include/villas/nodes/iec61850.hpp index d8cc7ec19..a63192ae5 100644 --- a/include/villas/nodes/iec61850.hpp +++ b/include/villas/nodes/iec61850.hpp @@ -19,8 +19,7 @@ #include #ifndef CONFIG_GOOSE_DEFAULT_DST_ADDRESS -#define CONFIG_GOOSE_DEFAULT_DST_ADDRESS \ - { 0x01, 0x0c, 0xcd, 0x01, 0x00, 0x01 } +#define CONFIG_GOOSE_DEFAULT_DST_ADDRESS {0x01, 0x0c, 0xcd, 0x01, 0x00, 0x01} #endif namespace villas { diff --git a/include/villas/nodes/test_rtt.hpp b/include/villas/nodes/test_rtt.hpp index 4732dda09..c398b1bdf 100644 --- a/include/villas/nodes/test_rtt.hpp +++ b/include/villas/nodes/test_rtt.hpp @@ -58,7 +58,7 @@ class TestRTT : public Node { : node(n), id(id), rate(rate), warmup(warmup), cooldown(cooldown), values(values), count(count), sent(0), received(0), missed(0), count_warmup(count_warmup), sent_warmup(0), received_warmup(0), - missed_warmup(0), filename(filename){}; + missed_warmup(0), filename(filename) {}; int start(); int stop(); @@ -95,7 +95,7 @@ class TestRTT : public Node { : Node(id, name), task(), formatter(nullptr), stream(nullptr), current(), shutdown(false) {} - virtual ~TestRTT(){}; + virtual ~TestRTT() {}; int prepare() override; diff --git a/lib/nodes/can.cpp b/lib/nodes/can.cpp index 784691ccf..489aa3723 100644 --- a/lib/nodes/can.cpp +++ b/lib/nodes/can.cpp @@ -300,20 +300,20 @@ static int can_conv_from_raw(union SignalData *sig, void *from, int size, case SignalType::INTEGER: switch (size) { case 1: - sig->i = (int64_t) * (int8_t *)from; + sig->i = (int64_t)*(int8_t *)from; return 0; case 2: - sig->i = (int64_t) * (int16_t *)from; + sig->i = (int64_t)*(int16_t *)from; return 0; case 3: - sig->i = (int64_t) * (int16_t *)from; - sig->i += ((int64_t) * ((int8_t *)(from) + 2)) << 16; + sig->i = (int64_t)*(int16_t *)from; + sig->i += ((int64_t)*((int8_t *)(from) + 2)) << 16; return 0; case 4: - sig->i = (int64_t) * (int32_t *)from; + sig->i = (int64_t)*(int32_t *)from; return 0; case 8: diff --git a/tests/unit/queue_signalled.cpp b/tests/unit/queue_signalled.cpp index 4d16a8983..8b9a59980 100644 --- a/tests/unit/queue_signalled.cpp +++ b/tests/unit/queue_signalled.cpp @@ -92,15 +92,15 @@ void *polled_consumer(void *ctx) { ParameterizedTestParameters(queue_signalled, simple) { static struct param params[] = { - {QueueSignalledMode::AUTO, 0, false}, - {QueueSignalledMode::PTHREAD, 0, false}, - {QueueSignalledMode::PTHREAD, 0, false}, - {QueueSignalledMode::PTHREAD, (int)QueueSignalledFlags::PROCESS_SHARED, - false}, - {QueueSignalledMode::POLLING, 0, false}, + {QueueSignalledMode::AUTO, 0, false}, + {QueueSignalledMode::PTHREAD, 0, false}, + {QueueSignalledMode::PTHREAD, 0, false}, + {QueueSignalledMode::PTHREAD, (int)QueueSignalledFlags::PROCESS_SHARED, + false}, + {QueueSignalledMode::POLLING, 0, false}, #if defined(__linux__) && defined(HAS_EVENTFD) - {QueueSignalledMode::EVENTFD, 0, false}, - {QueueSignalledMode::EVENTFD, 0, true} + {QueueSignalledMode::EVENTFD, 0, false}, + {QueueSignalledMode::EVENTFD, 0, true} #endif }; From 08896ef3fbab7e2e35870f32108b51302ad255b4 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Sun, 4 Jan 2026 22:32:51 +0100 Subject: [PATCH 08/14] fix (node-delta-sharing): Add integration file for node delta sharing Signed-off-by: Ritesh.K --- tests/integration/node-deltaSharing.sh | 191 +++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100755 tests/integration/node-deltaSharing.sh diff --git a/tests/integration/node-deltaSharing.sh b/tests/integration/node-deltaSharing.sh new file mode 100755 index 000000000..888a5b94d --- /dev/null +++ b/tests/integration/node-deltaSharing.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# +# Delta sharing node Integration Test +# +# Author: Ritesh Karki +# SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 + +echo "Test not ready" +exit 99 + +set -e + +DIR=$(mktemp -d) +pushd ${DIR} + +function finish { + popd + rm -rf ${DIR} +} +trap finish EXIT + +# Test data paths +TEST_PROFILE="${DIR}/open_delta_profile.json" +TEST_CACHE="${DIR}/delta_sharing_test_cache" +TEST_CONFIG="${DIR}/test_config.json" +TEST_OUTPUT="${DIR}/test_output.json" +TEST_INPUT="${DIR}/test_input.json" + + +# Set up test environment +function setup_test { + # Create test cache directory + mkdir -p "${TEST_CACHE}" + + cat > "${TEST_PROFILE}" << 'EOF' +{ + "shareCredentialsVersion": 1, + "endpoint": "https://sharing.delta.io/delta-sharing/", + "bearerToken": "faaie590d541265bcab1f2de9813274bf233" +} +EOF + + cat > "${TEST_INPUT}" << 'EOF' +{ + "nodes": { + "signal_source": { + "type": "signal", + "signal": "sine", + "rate": 10, + "limit": 5 + } + }, + "paths": [ + { + "in": "signal_source" + } + ] +} +EOF + + cat > "${TEST_CONFIG}" << EOF +{ + "nodes": { + "delta_reader": { + "type": "delta_sharing", + "profile_path": "${TEST_PROFILE}", + "cache_dir": "${TEST_CACHE}", + "table_path": "open-datasets.share#delta_sharing.default.COVID_19_NYT", + "op": "read", + "batch_size": 10 + }, + "delta_writer": { + "type": "delta_sharing", + "profile_path": "${TEST_PROFILE}", + "cache_dir": "${TEST_CACHE}", + "table_path": "open-delta-sharing.s3.us-west-2.amazonaws.com#samples.test_output", + "op": "write", + "batch_size": 10 + }, + "file1": { + "type": "file", + "uri": "${TEST_OUTPUT}", + "format": "json" + } + }, + "paths": [ + { + "in": "delta_reader", + "out": "file1" + } + ] +} +EOF + +} + +# Test 1: Verify Delta Sharing credentials +function test_credentials { + echo "Testing Delta Sharing credentials..." + + if [ ! -f "${TEST_PROFILE}" ]; then + log_error "Profile file not found: ${TEST_PROFILE}" + return 1 + fi + + # Check if profile has valid JSON structure + if ! python3 -m json.tool "${TEST_PROFILE}" > /dev/null 2>&1; then + log_error "Invalid JSON in profile file" + return 1 + fi + + + log_info "Credentials validation test passed" + return 0 +} + +# Test 2: Test Delta Sharing connection +function test_connection { + echo "Testing Delta Sharing server connection..." + + if timeout 2 "${VILLAS_NODE}" -c "${TEST_CONFIG}" --start 2>&1 | grep -q "Delta Sharing"; then + log_info "Connection test passed (Delta Sharing client initialized)" + return 0 + else + log_error "Connection test failed" + return 1 + fi +} + +# TODO: Test 3, to test data reading from Delta Sharing +function test_data_reading { + echo "Testing data reading from Delta Sharing..." + + echo "Attempting to read data from COVID-19_NYT table..." + + echo "Data reading test completed" + return 0 +} + +# Test 4: Test node configuration parsing +function test_config_parsing { + echo "Testing node configuration parsing..." + + if ! "${VILLAS_NODE}" --help | grep -i "delta_sharing"; then + echo "delta_sharing node type not found in villas-node" + return 1 + else + echo "delta_sharing node type present in villas-node" + fi + + #Test if the configuration can be parsed + if ! ("${VILLAS_NODE}" "${TEST_CONFIG}" &); then + echo "Node configuration check failed" + return 1 + else + echo "running example configuration for 3 seconds" + DELTA_PID=$! + kill $DELTA_PID + fi + # wait 3 + # kill $(DELTA_PID) + + echo "Configuration parsing test passed" + return 0 +} + +# Test 5: Test node lifecycle +function test_node_lifecycle { + echo "Testing node lifecycle..." + + if timeout 2 "${VILLAS_NODE}" -c "${TEST_CONFIG}" --start 2>&1 | grep -q "Delta Sharing\|Started"; then + echo "Node lifecycle test passed" + return 0 + else + echo "Node lifecycle test inconclusive" + return 0 + fi +} + +echo "Starting Delta Sharing integration tests with open datasets server..." + +# Run all tests +test_credentials || exit 1 +test_config_parsing || exit 1 +test_connection || exit 1 +test_data_reading || exit 1 +test_node_lifecycle || exit 1 + +echo "All tests passed!" +exit 0 From c4210e7ad35af632c98ab9de6f457bdc8572afa3 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Mon, 5 Jan 2026 16:16:05 +0100 Subject: [PATCH 09/14] fix (node-delta-sharing): Fix integration file for node delta-sharing Signed-off-by: Ritesh.K --- tests/integration/{node-deltaSharing.sh => node-delta_sharing.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/integration/{node-deltaSharing.sh => node-delta_sharing.sh} (100%) diff --git a/tests/integration/node-deltaSharing.sh b/tests/integration/node-delta_sharing.sh similarity index 100% rename from tests/integration/node-deltaSharing.sh rename to tests/integration/node-delta_sharing.sh From 0d4e0e2d56181ba43096b3be21eb2f0ba098c086 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Wed, 7 Jan 2026 14:16:25 +0000 Subject: [PATCH 10/14] fix (node-delta_sharing): Fix cppcheck errors Signed-off-by: Ritesh.K --- .../delta_sharing/delta_sharing_client.hpp | 14 ++++++------ .../delta_sharing_rest_client.hpp | 16 +++++++------- .../villas/nodes/delta_sharing/functions.hpp | 2 +- .../delta_sharing/delta_sharing_client.cpp | 16 +++++++++----- .../delta_sharing_rest_client.cpp | 22 ++++++++++--------- lib/nodes/delta_sharing/functions.cpp | 4 ++-- 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/include/villas/nodes/delta_sharing/delta_sharing_client.hpp b/include/villas/nodes/delta_sharing/delta_sharing_client.hpp index 9fda79b91..b43515e88 100644 --- a/include/villas/nodes/delta_sharing/delta_sharing_client.hpp +++ b/include/villas/nodes/delta_sharing/delta_sharing_client.hpp @@ -18,27 +18,27 @@ namespace DeltaSharing { struct DeltaSharingClient { public: - DeltaSharingClient(std::string filename, + DeltaSharingClient(const std::string &filename, std::optional cacheLocation); std::shared_ptr LoadAsArrowTable(std::string &url); std::shared_ptr ReadTableFromCache(std::string &url); const std::shared_ptr> - ListShares(int maxResult, std::string pageToken) const; + ListShares(int maxResult, const std::string &pageToken) const; const std::shared_ptr> ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, - std::string pageToken) const; + const std::string &pageToken) const; const std::shared_ptr> ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, - std::string pageToken) const; + const std::string &pageToken) const; const std::shared_ptr> ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, - std::string pageToken) const; + const std::string &pageToken) const; const std::shared_ptr> - ListFilesInTable(const DeltaSharingProtocol::Table) const; + ListFilesInTable(const DeltaSharingProtocol::Table &) const; const DeltaSharingProtocol::Metadata QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; const int GetNumberOfThreads() { return this->maxThreads; }; - void PopulateCache(std::string url) { + void PopulateCache(const std::string &url) { this->restClient.PopulateCache(url, this->cacheLocation); }; diff --git a/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp b/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp index ee0f6dbeb..543344064 100644 --- a/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp +++ b/include/villas/nodes/delta_sharing/delta_sharing_rest_client.hpp @@ -21,30 +21,30 @@ using json = nlohmann::json; namespace DeltaSharing { struct DeltaSharingRestClient { public: - DeltaSharingRestClient(std::string filename); + DeltaSharingRestClient(const std::string &filename); ~DeltaSharingRestClient(); const std::shared_ptr> - ListShares(int maxResult, std::string pageToken) const; + ListShares(int maxResult, const std::string &pageToken) const; const std::shared_ptr> ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, - std::string pageToken) const; + const std::string &pageToken) const; const std::shared_ptr> ListTables(const DeltaSharingProtocol::Schema &schema, int maxResult, - std::string pageToken) const; + const std::string &pageToken) const; const std::shared_ptr> ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, - std::string pageToken) const; + const std::string &pageToken) const; const std::shared_ptr> - ListFilesInTable(const DeltaSharingProtocol::Table) const; + ListFilesInTable(const DeltaSharingProtocol::Table &) const; const DeltaSharingProtocol::Metadata QueryTableMetadata(const DeltaSharingProtocol::Table &table) const; const DeltaSharingProtocol::DeltaSharingProfile &GetProfile() const; RestClient::Response get(std::string url); - void PopulateCache(std::string url, std::string cacheLocation); + void PopulateCache(const std::string &url, const std::string &cacheLocation); const bool shouldRetry(RestClient::Response &response) const; protected: - json ReadFromFile(std::string filename); + json ReadFromFile(const std::string &filename); private: DeltaSharingProtocol::DeltaSharingProfile profile; diff --git a/include/villas/nodes/delta_sharing/functions.hpp b/include/villas/nodes/delta_sharing/functions.hpp index 0ae89caa5..b448a9f10 100644 --- a/include/villas/nodes/delta_sharing/functions.hpp +++ b/include/villas/nodes/delta_sharing/functions.hpp @@ -18,7 +18,7 @@ namespace DeltaSharing { -const std::vector ParseURL(std::string path); +const std::vector ParseURL(const std::string &path); std::shared_ptr NewDeltaSharingClient(std::string profile, std::optional cacheLocation); diff --git a/lib/nodes/delta_sharing/delta_sharing_client.cpp b/lib/nodes/delta_sharing/delta_sharing_client.cpp index 79283f3c8..59b15ee34 100644 --- a/lib/nodes/delta_sharing/delta_sharing_client.cpp +++ b/lib/nodes/delta_sharing/delta_sharing_client.cpp @@ -26,7 +26,7 @@ #include namespace DeltaSharing { -DeltaSharingClient::DeltaSharingClient(std::string filename, +DeltaSharingClient::DeltaSharingClient(const std::string &filename, std::optional cacheLocation) : restClient(filename) { auto path = std::filesystem::current_path().generic_string(); @@ -165,31 +165,35 @@ DeltaSharingClient::ReadTableFromCache(std::string &completePath) { }; const std::shared_ptr> -DeltaSharingClient::ListShares(int maxResult, std::string pageToken) const { +DeltaSharingClient::ListShares(int maxResult, + const std::string &pageToken) const { return this->restClient.ListShares(maxResult, pageToken); }; const std::shared_ptr> DeltaSharingClient::ListSchemas(const DeltaSharingProtocol::Share &share, - int maxResult, std::string pageToken) const { + int maxResult, + const std::string &pageToken) const { return this->restClient.ListSchemas(share, maxResult, pageToken); }; const std::shared_ptr> DeltaSharingClient::ListTables(const DeltaSharingProtocol::Schema &schema, - int maxResult, std::string pageToken) const { + int maxResult, + const std::string &pageToken) const { return this->restClient.ListTables(schema, maxResult, pageToken); }; const std::shared_ptr> DeltaSharingClient::ListAllTables(const DeltaSharingProtocol::Share &share, - int maxResult, std::string pageToken) const { + int maxResult, + const std::string &pageToken) const { return this->restClient.ListAllTables(share, maxResult, pageToken); }; const std::shared_ptr> DeltaSharingClient::ListFilesInTable( - const DeltaSharingProtocol::Table table) const { + const DeltaSharingProtocol::Table &table) const { return this->restClient.ListFilesInTable(table); }; diff --git a/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp b/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp index 79191ddc0..617509bcc 100644 --- a/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp +++ b/lib/nodes/delta_sharing/delta_sharing_rest_client.cpp @@ -23,7 +23,7 @@ namespace DeltaSharing { const std::string DeltaSharingRestClient::user_agent = "delta-sharing-CPP/0.0.1"; -DeltaSharingRestClient::DeltaSharingRestClient(std::string filename) { +DeltaSharingRestClient::DeltaSharingRestClient(const std::string &filename) { json j = ReadFromFile(filename); if (j.empty()) { return; @@ -37,7 +37,8 @@ DeltaSharingRestClient::~DeltaSharingRestClient() { }; const std::shared_ptr> -DeltaSharingRestClient::ListShares(int maxResult, std::string pageToken) const { +DeltaSharingRestClient::ListShares(int maxResult, + const std::string &pageToken) const { std::unique_ptr c = std::unique_ptr( new RestClient::Connection(this->profile.endpoint)); @@ -57,7 +58,7 @@ DeltaSharingRestClient::ListShares(int maxResult, std::string pageToken) const { const std::shared_ptr> DeltaSharingRestClient::ListSchemas(const DeltaSharingProtocol::Share &share, int maxResult, - std::string pageToken) const { + const std::string &pageToken) const { std::unique_ptr c = std::unique_ptr( new RestClient::Connection(this->profile.endpoint)); @@ -78,7 +79,8 @@ DeltaSharingRestClient::ListSchemas(const DeltaSharingProtocol::Share &share, const std::shared_ptr> DeltaSharingRestClient::ListTables(const DeltaSharingProtocol::Schema &schema, - int maxResult, std::string pageToken) const { + int maxResult, + const std::string &pageToken) const { std::unique_ptr c = std::unique_ptr( new RestClient::Connection(this->profile.endpoint)); @@ -100,7 +102,7 @@ DeltaSharingRestClient::ListTables(const DeltaSharingProtocol::Schema &schema, const std::shared_ptr> DeltaSharingRestClient::ListAllTables(const DeltaSharingProtocol::Share &share, int maxResult, - std::string pageToken) const { + const std::string &pageToken) const { std::unique_ptr c = std::unique_ptr( new RestClient::Connection(this->profile.endpoint)); @@ -142,7 +144,7 @@ const DeltaSharingProtocol::Metadata DeltaSharingRestClient::QueryTableMetadata( return m; }; -json DeltaSharingRestClient::ReadFromFile(std::string filename) { +json DeltaSharingRestClient::ReadFromFile(const std::string &filename) { std::ifstream is; try { is.open(filename, std::ifstream::in); @@ -162,8 +164,8 @@ DeltaSharingRestClient::GetProfile() const { return this->profile; } -void DeltaSharingRestClient::PopulateCache(std::string url, - std::string cacheLocation) { +void DeltaSharingRestClient::PopulateCache(const std::string &url, + const std::string &cacheLocation) { int protocolLength = 0; if ((url.find("http://")) != std::string::npos) { protocolLength = 7; @@ -229,7 +231,7 @@ void DeltaSharingRestClient::PopulateCache(std::string url, const std::shared_ptr> DeltaSharingRestClient::ListFilesInTable( - DeltaSharingProtocol::Table table) const { + const DeltaSharingProtocol::Table &table) const { std::unique_ptr c = std::unique_ptr( new RestClient::Connection(this->profile.endpoint)); @@ -240,7 +242,7 @@ DeltaSharingRestClient::ListFilesInTable( h.insert({"Content-Type", "application/json; charset=UTF-8"}); h.insert({"Authorization", "Bearer: " + this->profile.bearerToken}); c->SetHeaders(h); - DeltaSharingProtocol::data d; + DeltaSharingProtocol::data d{}; json j = d; RestClient::Response r = c->post(path, j.dump()); int cnt = 0; diff --git a/lib/nodes/delta_sharing/functions.cpp b/lib/nodes/delta_sharing/functions.cpp index 2bdc7ebcd..157a5bc6a 100644 --- a/lib/nodes/delta_sharing/functions.cpp +++ b/lib/nodes/delta_sharing/functions.cpp @@ -12,7 +12,7 @@ namespace DeltaSharing { -const std::vector ParseURL(std::string path) { +const std::vector ParseURL(const std::string &path) { std::vector urlparts; std::string url = path; auto pos = url.find_last_of('#'); @@ -42,7 +42,7 @@ NewDeltaSharingClient(std::string profile, return std::make_shared(profile, cacheLocation); }; -const std::shared_ptr LoadAsArrowTable(std::string path, +const std::shared_ptr LoadAsArrowTable(const std::string &path, int fileno) { auto p = ParseURL(path); From 3e2f5a7f6f3c5d59e8d7ded6aa4a404b0d055002 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Wed, 7 Jan 2026 14:17:06 +0000 Subject: [PATCH 11/14] fix (node-delta-sharing): Add example conf for delta_sharing node Signed-off-by: Ritesh.K --- etc/examples/nodes/delta_sharing.conf | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 etc/examples/nodes/delta_sharing.conf diff --git a/etc/examples/nodes/delta_sharing.conf b/etc/examples/nodes/delta_sharing.conf new file mode 100644 index 000000000..c08c87910 --- /dev/null +++ b/etc/examples/nodes/delta_sharing.conf @@ -0,0 +1,29 @@ +nodes = { + delta_reader = { + type = "delta_sharing" + profile_path = "${TEST_PROFILE}" + cache_dir = "${TEST_CACHE}" + table_path = "open-datasets.share#delta_sharing.default.COVID_19_NYT", + op = "read" + batch_size = 10 + }, + delta_writer = { + type = "delta_sharing", + profile_path = "${TEST_PROFILE}" + cache_dir = "${TEST_CACHE}" + table_path = "open-delta-sharing.s3.us-west-2.amazonaws.com#samples.test_output", + op = "write" + batch_size = 10 + }, + file1 = { + type = "file" + uri = "${TEST_OUTPUT}" + format = "json" + } +}, +paths = [ + { + in = "delta_reader" + out = "file1" + } +] From 9b6859646cd0efa7a902150595ba78c6127f343b Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Wed, 7 Jan 2026 16:46:54 +0100 Subject: [PATCH 12/14] fix (node-delta-sharing): Fix syntax error in delta_sharing example config Signed-off-by: Ritesh.K --- etc/examples/nodes/delta_sharing.conf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etc/examples/nodes/delta_sharing.conf b/etc/examples/nodes/delta_sharing.conf index c08c87910..2868e2878 100644 --- a/etc/examples/nodes/delta_sharing.conf +++ b/etc/examples/nodes/delta_sharing.conf @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 nodes = { delta_reader = { type = "delta_sharing" @@ -20,10 +22,10 @@ nodes = { uri = "${TEST_OUTPUT}" format = "json" } -}, -paths = [ +} +paths = ( { in = "delta_reader" out = "file1" } -] +) From 5b8f0e366ff1d064ae7c2f807dfaab3f446386e0 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Wed, 7 Jan 2026 17:06:08 +0100 Subject: [PATCH 13/14] fix (node-delta-sharing): Fix error in delta_sharing example config Signed-off-by: Ritesh.K --- etc/examples/nodes/delta_sharing.conf | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/etc/examples/nodes/delta_sharing.conf b/etc/examples/nodes/delta_sharing.conf index 2868e2878..496a5a400 100644 --- a/etc/examples/nodes/delta_sharing.conf +++ b/etc/examples/nodes/delta_sharing.conf @@ -3,23 +3,15 @@ nodes = { delta_reader = { type = "delta_sharing" - profile_path = "${TEST_PROFILE}" - cache_dir = "${TEST_CACHE}" + profile_path = "." + cache_dir = "." table_path = "open-datasets.share#delta_sharing.default.COVID_19_NYT", op = "read" batch_size = 10 }, - delta_writer = { - type = "delta_sharing", - profile_path = "${TEST_PROFILE}" - cache_dir = "${TEST_CACHE}" - table_path = "open-delta-sharing.s3.us-west-2.amazonaws.com#samples.test_output", - op = "write" - batch_size = 10 - }, file1 = { type = "file" - uri = "${TEST_OUTPUT}" + uri = "." format = "json" } } From 6ad4419c761b3d1f7a7eb850128f35e08447bdf6 Mon Sep 17 00:00:00 2001 From: "Ritesh.K" Date: Mon, 12 Jan 2026 16:38:10 +0100 Subject: [PATCH 14/14] fix (node-delta-sharing): Fix cpp file format for CI pre-commit tests Signed-off-by: Ritesh.K --- common/include/villas/kernel/devices/device.hpp | 2 +- .../include/villas/kernel/devices/ip_device.hpp | 2 +- .../villas/kernel/devices/linux_driver.hpp | 4 ++-- .../villas/kernel/devices/platform_device.hpp | 4 ++-- common/include/villas/utils.hpp | 8 ++++---- common/lib/kernel/devices/device_connection.cpp | 2 +- fpga/include/villas/fpga/ips/i2c.hpp | 5 +++-- fpga/include/villas/fpga/platform_card.hpp | 2 +- fpga/include/villas/fpga/utils.hpp | 6 +++--- fpga/lib/core_connection.cpp | 2 +- fpga/src/villas-fpga-pipe.cpp | 2 +- include/villas/nodes/iec61850.hpp | 3 ++- include/villas/nodes/test_rtt.hpp | 4 ++-- lib/nodes/can.cpp | 10 +++++----- tests/unit/queue_signalled.cpp | 16 ++++++++-------- 15 files changed, 37 insertions(+), 35 deletions(-) diff --git a/common/include/villas/kernel/devices/device.hpp b/common/include/villas/kernel/devices/device.hpp index a3b1a8656..d6602da10 100644 --- a/common/include/villas/kernel/devices/device.hpp +++ b/common/include/villas/kernel/devices/device.hpp @@ -19,7 +19,7 @@ namespace devices { class Device { public: - virtual ~Device() {}; + virtual ~Device(){}; virtual std::optional> driver() const = 0; virtual std::optional iommu_group() const = 0; diff --git a/common/include/villas/kernel/devices/ip_device.hpp b/common/include/villas/kernel/devices/ip_device.hpp index 374383e78..4ec192f08 100644 --- a/common/include/villas/kernel/devices/ip_device.hpp +++ b/common/include/villas/kernel/devices/ip_device.hpp @@ -25,7 +25,7 @@ class IpDevice : public PlatformDevice { private: IpDevice() = delete; IpDevice(const fs::path valid_path) //! Dont allow unvalidated paths - : PlatformDevice(valid_path) {}; + : PlatformDevice(valid_path){}; public: size_t addr() const; diff --git a/common/include/villas/kernel/devices/linux_driver.hpp b/common/include/villas/kernel/devices/linux_driver.hpp index 74dadb4f0..b1c01cb14 100644 --- a/common/include/villas/kernel/devices/linux_driver.hpp +++ b/common/include/villas/kernel/devices/linux_driver.hpp @@ -33,11 +33,11 @@ class LinuxDriver : public Driver { public: LinuxDriver(const fs::path path) : LinuxDriver(path, path / fs::path(BIND_DEFAULT), - path / fs::path(UNBIND_DEFAULT)) {}; + path / fs::path(UNBIND_DEFAULT)){}; LinuxDriver(const fs::path path, const fs::path bind_path, const fs::path unbind_path) - : path(path), bind_path(bind_path), unbind_path(unbind_path) {}; + : path(path), bind_path(bind_path), unbind_path(unbind_path){}; public: void attach(const Device &device) const override; diff --git a/common/include/villas/kernel/devices/platform_device.hpp b/common/include/villas/kernel/devices/platform_device.hpp index bad32f2cb..1071a5a84 100644 --- a/common/include/villas/kernel/devices/platform_device.hpp +++ b/common/include/villas/kernel/devices/platform_device.hpp @@ -29,12 +29,12 @@ class PlatformDevice : public Device { public: PlatformDevice(const fs::path path) : PlatformDevice(path, fs::path(PROBE_DEFAULT), - path / fs::path(OVERRIDE_DEFAULT)) {}; + path / fs::path(OVERRIDE_DEFAULT)){}; PlatformDevice(const fs::path path, const fs::path probe_path, const fs::path override_path) : m_path(path), m_probe_path(probe_path), - m_override_path(override_path) {}; + m_override_path(override_path){}; // Implement device interface std::optional> driver() const override; diff --git a/common/include/villas/utils.hpp b/common/include/villas/utils.hpp index 0a0ad01e9..1ebb1cf08 100644 --- a/common/include/villas/utils.hpp +++ b/common/include/villas/utils.hpp @@ -52,7 +52,7 @@ #ifdef ALIGN #undef ALIGN #endif -#define ALIGN(x, a) ALIGN_MASK(x, (uintptr_t)(a) - 1) +#define ALIGN(x, a) ALIGN_MASK(x, (uintptr_t)(a)-1) #define ALIGN_MASK(x, m) (((uintptr_t)(x) + (m)) & ~(m)) #define IS_ALIGNED(x, a) (ALIGN(x, a) == (uintptr_t)x) @@ -64,13 +64,13 @@ } while (0) // Round-up integer division -#define CEIL(x, y) (((x) + (y) - 1) / (y)) +#define CEIL(x, y) (((x) + (y)-1) / (y)) // Get nearest up-rounded power of 2 -#define LOG2_CEIL(x) (1 << (villas::utils::log2i((x) - 1) + 1)) +#define LOG2_CEIL(x) (1 << (villas::utils::log2i((x)-1) + 1)) // Check if the number is a power of 2 -#define IS_POW2(x) (((x) != 0) && !((x) & ((x) - 1))) +#define IS_POW2(x) (((x) != 0) && !((x) & ((x)-1))) // Calculate the number of elements in an array. #define ARRAY_LEN(a) (sizeof(a) / sizeof(a)[0]) diff --git a/common/lib/kernel/devices/device_connection.cpp b/common/lib/kernel/devices/device_connection.cpp index 28b156e79..b32872f02 100644 --- a/common/lib/kernel/devices/device_connection.cpp +++ b/common/lib/kernel/devices/device_connection.cpp @@ -18,7 +18,7 @@ namespace devices { DeviceConnection::DeviceConnection( std::shared_ptr vfio_device) - : logger(villas::Log::get("DeviceConnection")), vfio_device(vfio_device) {}; + : logger(villas::Log::get("DeviceConnection")), vfio_device(vfio_device){}; DeviceConnection DeviceConnection::from( const villas::kernel::devices::Device &device, diff --git a/fpga/include/villas/fpga/ips/i2c.hpp b/fpga/include/villas/fpga/ips/i2c.hpp index 1854b1cf8..b90f34fac 100644 --- a/fpga/include/villas/fpga/ips/i2c.hpp +++ b/fpga/include/villas/fpga/ips/i2c.hpp @@ -20,7 +20,8 @@ namespace fpga { namespace ip { #define I2C_SWTICH_ADDR 0x70 -#define I2C_SWITCH_CHANNEL_MAP {0x20, 0x80, 0x02, 0x08, 0x10, 0x40, 0x01, 0x04} +#define I2C_SWITCH_CHANNEL_MAP \ + { 0x20, 0x80, 0x02, 0x08, 0x10, 0x40, 0x01, 0x04 } #define I2C_IOEXT_ADDR 0x20 #define I2C_IOEXT_REG_DIR 0x03 #define I2C_IOEXT_REG_OUT 0x01 @@ -47,7 +48,7 @@ class I2c : public Node { public: Switch(I2c *i2c, uint8_t address, Logger logger = villas::Log::get("i2c")) : i2c(i2c), address(address), channel(0), readOnce(false), switchLock(), - logger(logger) {}; + logger(logger){}; Switch(const Switch &other) = delete; Switch &operator=(const Switch &other) = delete; void setChannel(uint8_t channel); diff --git a/fpga/include/villas/fpga/platform_card.hpp b/fpga/include/villas/fpga/platform_card.hpp index e122caf0d..22fc4b93b 100644 --- a/fpga/include/villas/fpga/platform_card.hpp +++ b/fpga/include/villas/fpga/platform_card.hpp @@ -25,7 +25,7 @@ class PlatformCard : public Card { public: PlatformCard(std::shared_ptr vfioContainer); - ~PlatformCard() {}; + ~PlatformCard(){}; std::vector core_connections; diff --git a/fpga/include/villas/fpga/utils.hpp b/fpga/include/villas/fpga/utils.hpp index f7025b990..d8493063c 100644 --- a/fpga/include/villas/fpga/utils.hpp +++ b/fpga/include/villas/fpga/utils.hpp @@ -90,7 +90,7 @@ class BufferedSampleFormatter { BufferedSampleFormatter(const size_t bufSamples, const size_t bufSampleSize) : buf(bufSamples * bufSampleSize + 1), // Leave room for a final `\0' bufSamples(bufSamples), bufSampleSize(bufSampleSize), - currentBufLoc(0) {}; + currentBufLoc(0){}; BufferedSampleFormatter() = delete; BufferedSampleFormatter(const BufferedSampleFormatter &) = delete; virtual char *nextBufPos() { return &buf[(currentBufLoc++) * bufSampleSize]; } @@ -99,7 +99,7 @@ class BufferedSampleFormatter { class BufferedSampleFormatterShort : public BufferedSampleFormatter { public: BufferedSampleFormatterShort(size_t bufSizeInSamples) - : BufferedSampleFormatter(bufSizeInSamples, formatStringSize) {}; + : BufferedSampleFormatter(bufSizeInSamples, formatStringSize){}; virtual void format(float value) override { size_t chars; @@ -120,7 +120,7 @@ class BufferedSampleFormatterLong : public BufferedSampleFormatter { public: BufferedSampleFormatterLong(size_t bufSizeInSamples) : BufferedSampleFormatter(bufSizeInSamples, formatStringSize), - sampleCnt(0) {}; + sampleCnt(0){}; virtual void format(float value) override { if (std::snprintf(nextBufPos(), formatStringSize + 1, formatString, diff --git a/fpga/lib/core_connection.cpp b/fpga/lib/core_connection.cpp index c70d96536..242009a34 100644 --- a/fpga/lib/core_connection.cpp +++ b/fpga/lib/core_connection.cpp @@ -20,7 +20,7 @@ CoreConnection::CoreConnection( std::shared_ptr ip, villas::kernel::devices::DeviceConnection device_connection) : logger(villas::Log::get("CoreConnection")), ip(ip), - device_connection(device_connection) {}; + device_connection(device_connection){}; CoreConnection CoreConnection::from(std::shared_ptr ip, diff --git a/fpga/src/villas-fpga-pipe.cpp b/fpga/src/villas-fpga-pipe.cpp index 0b18ba1ae..3aba1657a 100644 --- a/fpga/src/villas-fpga-pipe.cpp +++ b/fpga/src/villas-fpga-pipe.cpp @@ -78,7 +78,7 @@ int main(int argc, char *argv[]) { for (auto aurora : aurora_channels) aurora->dump(); - // Configure Crossbar switch + // Configure Crossbar switch #if 1 aurora_channels[3]->connect(aurora_channels[3]->getDefaultMasterPort(), dma->getDefaultSlavePort()); diff --git a/include/villas/nodes/iec61850.hpp b/include/villas/nodes/iec61850.hpp index a63192ae5..d8cc7ec19 100644 --- a/include/villas/nodes/iec61850.hpp +++ b/include/villas/nodes/iec61850.hpp @@ -19,7 +19,8 @@ #include #ifndef CONFIG_GOOSE_DEFAULT_DST_ADDRESS -#define CONFIG_GOOSE_DEFAULT_DST_ADDRESS {0x01, 0x0c, 0xcd, 0x01, 0x00, 0x01} +#define CONFIG_GOOSE_DEFAULT_DST_ADDRESS \ + { 0x01, 0x0c, 0xcd, 0x01, 0x00, 0x01 } #endif namespace villas { diff --git a/include/villas/nodes/test_rtt.hpp b/include/villas/nodes/test_rtt.hpp index c398b1bdf..4732dda09 100644 --- a/include/villas/nodes/test_rtt.hpp +++ b/include/villas/nodes/test_rtt.hpp @@ -58,7 +58,7 @@ class TestRTT : public Node { : node(n), id(id), rate(rate), warmup(warmup), cooldown(cooldown), values(values), count(count), sent(0), received(0), missed(0), count_warmup(count_warmup), sent_warmup(0), received_warmup(0), - missed_warmup(0), filename(filename) {}; + missed_warmup(0), filename(filename){}; int start(); int stop(); @@ -95,7 +95,7 @@ class TestRTT : public Node { : Node(id, name), task(), formatter(nullptr), stream(nullptr), current(), shutdown(false) {} - virtual ~TestRTT() {}; + virtual ~TestRTT(){}; int prepare() override; diff --git a/lib/nodes/can.cpp b/lib/nodes/can.cpp index 489aa3723..784691ccf 100644 --- a/lib/nodes/can.cpp +++ b/lib/nodes/can.cpp @@ -300,20 +300,20 @@ static int can_conv_from_raw(union SignalData *sig, void *from, int size, case SignalType::INTEGER: switch (size) { case 1: - sig->i = (int64_t)*(int8_t *)from; + sig->i = (int64_t) * (int8_t *)from; return 0; case 2: - sig->i = (int64_t)*(int16_t *)from; + sig->i = (int64_t) * (int16_t *)from; return 0; case 3: - sig->i = (int64_t)*(int16_t *)from; - sig->i += ((int64_t)*((int8_t *)(from) + 2)) << 16; + sig->i = (int64_t) * (int16_t *)from; + sig->i += ((int64_t) * ((int8_t *)(from) + 2)) << 16; return 0; case 4: - sig->i = (int64_t)*(int32_t *)from; + sig->i = (int64_t) * (int32_t *)from; return 0; case 8: diff --git a/tests/unit/queue_signalled.cpp b/tests/unit/queue_signalled.cpp index 8b9a59980..4d16a8983 100644 --- a/tests/unit/queue_signalled.cpp +++ b/tests/unit/queue_signalled.cpp @@ -92,15 +92,15 @@ void *polled_consumer(void *ctx) { ParameterizedTestParameters(queue_signalled, simple) { static struct param params[] = { - {QueueSignalledMode::AUTO, 0, false}, - {QueueSignalledMode::PTHREAD, 0, false}, - {QueueSignalledMode::PTHREAD, 0, false}, - {QueueSignalledMode::PTHREAD, (int)QueueSignalledFlags::PROCESS_SHARED, - false}, - {QueueSignalledMode::POLLING, 0, false}, + {QueueSignalledMode::AUTO, 0, false}, + {QueueSignalledMode::PTHREAD, 0, false}, + {QueueSignalledMode::PTHREAD, 0, false}, + {QueueSignalledMode::PTHREAD, (int)QueueSignalledFlags::PROCESS_SHARED, + false}, + {QueueSignalledMode::POLLING, 0, false}, #if defined(__linux__) && defined(HAS_EVENTFD) - {QueueSignalledMode::EVENTFD, 0, false}, - {QueueSignalledMode::EVENTFD, 0, true} + {QueueSignalledMode::EVENTFD, 0, false}, + {QueueSignalledMode::EVENTFD, 0, true} #endif };