From 45004a005caacfe93d05ebaa472c12094b02356c Mon Sep 17 00:00:00 2001 From: Philipp Jungkamp Date: Mon, 9 Feb 2026 16:38:44 +0100 Subject: [PATCH 1/2] utils: Add glob helper function Signed-off-by: Philipp Jungkamp --- common/include/villas/utils.hpp | 5 +++ common/lib/utils.cpp | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/common/include/villas/utils.hpp b/common/include/villas/utils.hpp index 1ebb1cf08..ae74ff01f 100644 --- a/common/include/villas/utils.hpp +++ b/common/include/villas/utils.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -212,6 +213,10 @@ template struct overloaded : Ts... { // Explicit deduction guide (not needed as of C++20) template overloaded(Ts...) -> overloaded; +// glob-style filesystem pattern matching +std::vector glob(fs::path const &pattern, + std::span searchDirectories); + void write_to_file(std::string data, const fs::path file); std::vector read_names_in_directory(const fs::path &directory); diff --git a/common/lib/utils.cpp b/common/lib/utils.cpp index 1431533eb..93e21e766 100644 --- a/common/lib/utils.cpp +++ b/common/lib/utils.cpp @@ -12,12 +12,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -352,6 +354,70 @@ bool isPrivileged() { return true; } +// internal glob implementation details +namespace { +bool isGlobPattern(fs::path const &path) { + static const auto specialCharacters = fs::path("?*[").native(); + auto const &string = path.native(); + return std::ranges::find_first_of(string, specialCharacters) != string.end(); +} + +bool isGlobMatch(fs::path const &pattern, fs::path const &path) { + return ::fnmatch(pattern.c_str(), path.c_str(), FNM_PATHNAME) == 0; +} + +void globImpl(std::vector &result, fs::path &&path, + std::ranges::subrange pattern) { + [[maybe_unused]] auto discardErrorCode = std::error_code{}; + + if (pattern.empty()) { + // we've reached the end of our pattern + if (fs::exists(path, discardErrorCode)) + result.push_back(path); + return; + } + + if (not fs::is_directory(path, discardErrorCode)) + return; + + if (not isGlobPattern(pattern.front())) { + path /= pattern.front(); + return globImpl(result, std::move(path), std::move(pattern).next()); + } else { + auto nextPattern = pattern.next(); + for (auto entry : fs::directory_iterator(path)) { + if (not isGlobMatch(pattern.front(), entry.path().filename())) + continue; + + globImpl(result, fs::path(entry.path()), nextPattern); + } + } +} +} // namespace + +std::vector glob(fs::path const &pattern, + std::span searchDirectories) { + auto logger = Log::get("glob"); + std::vector result; + if (pattern.is_absolute()) { + logger->debug("Matching absolute pattern {:?}", pattern.string()); + globImpl(result, pattern.root_path(), pattern); + } else { + for (auto path : searchDirectories) { + logger->debug("Matching relative pattern {:?} in {:?}", pattern.string(), + path.string()); + globImpl(result, std::move(path), pattern); + } + } + + if (result.empty()) { + throw std::runtime_error( + fmt::format("Could not find any file matching {:?}", pattern.string())); + } + + return result; +} + void write_to_file(std::string data, const fs::path file) { villas::Log::get("Filewriter")->debug("{} > {}", data, file.string()); std::ofstream outputFile(file.string()); From 4037b5d47a967c4677c5b1fcfd30ea585872f5ea Mon Sep 17 00:00:00 2001 From: Philipp Jungkamp Date: Mon, 26 Jan 2026 16:06:37 +0100 Subject: [PATCH 2/2] Parse configuration files using nlohmann::json Signed-off-by: Philipp Jungkamp --- CMakeLists.txt | 1 + include/villas/config_class.hpp | 64 +--- include/villas/json.hpp | 61 ++++ include/villas/json_fwd.hpp | 30 ++ include/villas/super_node.hpp | 2 +- lib/CMakeLists.txt | 2 + lib/config.cpp | 359 ++-------------------- lib/json.cpp | 525 ++++++++++++++++++++++++++++++++ lib/nodes/fpga.cpp | 3 +- packaging/nix/villas.nix | 26 +- tests/unit/config.cpp | 124 ++++---- 11 files changed, 742 insertions(+), 455 deletions(-) create mode 100644 include/villas/json.hpp create mode 100644 include/villas/json_fwd.hpp create mode 100644 lib/json.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8272382e3..1990aba05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ endif() # Check packages find_package(PkgConfig REQUIRED) find_package(Threads REQUIRED) +find_package(nlohmann_json REQUIRED) find_package(OpenMP) find_package(IBVerbs) find_package(RDMACM) diff --git a/include/villas/config_class.hpp b/include/villas/config_class.hpp index e6b372b9c..e8d96ed2d 100644 --- a/include/villas/config_class.hpp +++ b/include/villas/config_class.hpp @@ -8,7 +8,6 @@ #pragma once #include -#include #include #include @@ -25,71 +24,26 @@ namespace villas { namespace node { class Config { - -protected: - using str_walk_fcn_t = std::function; - +private: Logger logger; - - std::list includeDirectories; - std::string configPath; - - // Check if file exists on local system. - static bool isLocalFile(const std::string &uri) { - return access(uri.c_str(), F_OK) != -1; - } - - // Decode configuration file. - json_t *decode(FILE *f); - -#ifdef WITH_CONFIG - // Convert libconfig .conf file to libjansson .json file. - json_t *libconfigDecode(FILE *f); - - static const char **includeFuncStub(config_t *cfg, const char *include_dir, - const char *path, const char **error); - - const char **includeFunc(config_t *cfg, const char *include_dir, - const char *path, const char **error); -#endif // WITH_CONFIG - - // Load configuration from standard input (stdim). - FILE *loadFromStdio(); - - // Load configuration from local file. - FILE *loadFromLocalFile(const std::string &u); - - std::list resolveIncludes(const std::string &name); - - void resolveEnvVars(std::string &text); - - // Resolve custom include directives. - json_t *expandIncludes(json_t *in); - - // To shell-like subsitution of environment variables in strings. - json_t *expandEnvVars(json_t *in); - - // Run a callback function for each string in the config - json_t *walkStrings(json_t *in, str_walk_fcn_t cb); - - // Get the include dirs - std::list getIncludeDirectories(FILE *f) const; + fs::path configPath; public: json_t *root; Config(); - Config(const std::string &u); + Config(fs::path path); + Config(Config const &) = delete; + Config &operator=(Config const &) = delete; + Config(Config &&) = delete; + Config &operator=(Config &&) = delete; ~Config(); - json_t *load(std::FILE *f, bool resolveIncludes = true, - bool resolveEnvVars = true); - - json_t *load(const std::string &u, bool resolveIncludes = true, + json_t *load(fs::path path, bool resolveIncludes = true, bool resolveEnvVars = true); - std::string const &getConfigPath() const { return configPath; } + fs::path const &getConfigPath() const { return configPath; } }; } // namespace node diff --git a/include/villas/json.hpp b/include/villas/json.hpp new file mode 100644 index 000000000..9c14efbdc --- /dev/null +++ b/include/villas/json.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include + +#include +#include + +#include +#include + +// libjansson forward declaration +struct json_t; + +namespace villas { + +// deleter for libjansson ::json_t * values +struct libjansson_deleter { + void operator()(::json_t *) const; +}; + +// smart pointer for libjansson ::json_t * values. +using libjansson_ptr = std::unique_ptr<::json_t, libjansson_deleter>; + +// base class which injects VILLASnode specific functionality +// into the config_json specialization of nlohmann::basic_json. +class config_json_base { +private: + friend config_json; + config_json_base() = default; + +public: + // libjansson compatability + static config_json from_libjansson(::json_t const *); + libjansson_ptr to_libjansson() const; + + // configuration parsing options + struct options_t { + // expand $ENV variable substitutions + bool expand_substitutions = false; + // expand $include keys + bool expand_includes = false; + // expand deprecated ${ENV} substitutions + bool expand_deprecated = false; + // a set of base directories to search for includes + std::span include_directories = {}; + }; + + // load a VILLASnode configuration + static config_json load_config(std::FILE *, options_t); + static config_json load_config(std::string_view, options_t); + static config_json load_config_file(fs::path const &, options_t); +}; + +}; // namespace villas + +template <> // format config_json using operator<< +struct fmt::formatter : ostream_formatter {}; + +template <> // format config_json_pointer using operator<< +struct fmt::formatter : ostream_formatter {}; diff --git a/include/villas/json_fwd.hpp b/include/villas/json_fwd.hpp new file mode 100644 index 000000000..04ba9e760 --- /dev/null +++ b/include/villas/json_fwd.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include + +struct json_t; + +namespace villas { + +class config_json_base; + +using config_json = + nlohmann::basic_json, // BinaryType + config_json_base // CustomBaseClass + >; + +using config_json_pointer = nlohmann::json_pointer; + +}; // namespace villas diff --git a/include/villas/super_node.hpp b/include/villas/super_node.hpp index f4836fdfd..12b6de929 100644 --- a/include/villas/super_node.hpp +++ b/include/villas/super_node.hpp @@ -140,7 +140,7 @@ class SuperNode { json_t *getConfig() { return config.root; } - const std::string &getConfigPath() const { return config.getConfigPath(); } + fs::path const &getConfigPath() const { return config.getConfigPath(); } int getAffinity() const { return affinity; } diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index e95215e5a..238d3dddf 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -13,6 +13,7 @@ list(APPEND INCLUDE_DIRS set(LIBRARIES villas-common + nlohmann_json::nlohmann_json PkgConfig::JANSSON PkgConfig::UUID m @@ -26,6 +27,7 @@ set(LIB_SRC config.cpp dumper.cpp format.cpp + json.cpp mapping.cpp mapping_list.cpp memory.cpp diff --git a/lib/config.cpp b/lib/config.cpp index 0bab007ef..7c52e9a43 100644 --- a/lib/config.cpp +++ b/lib/config.cpp @@ -5,9 +5,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include +#include +#include #include +#include #include #include #include @@ -16,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -27,342 +30,34 @@ using namespace villas; using namespace villas::node; +using namespace std::string_view_literals; Config::Config() : logger(Log::get("config")), root(nullptr) {} -Config::Config(const std::string &u) : Config() { root = load(u); } +Config::Config(fs::path path) : Config() { root = load(std::move(path)); } Config::~Config() { json_decref(root); } -json_t *Config::load(std::FILE *f, bool resolveInc, bool resolveEnvVars) { - json_t *root = decode(f); - - if (resolveInc) { - json_t *root_old = root; - root = expandIncludes(root); - json_decref(root_old); - } - - if (resolveEnvVars) { - json_t *root_old = root; - root = expandEnvVars(root); - json_decref(root_old); +json_t *Config::load(fs::path path, bool resolveInc, bool resolveEnvVars) { + if (path == "-") { + logger->info("Reading configuration from standard input"); + configPath = fs::current_path(); + auto opts = config_json::options_t{ + .expand_substitutions = resolveEnvVars, + .expand_includes = resolveInc, + .expand_deprecated = true, + .include_directories = std::span{&configPath, 1}, + }; + return config_json::load_config(stdin, opts).to_libjansson().release(); + } else { + logger->info("Reading configuration from file: {}", path.string()); + configPath = fs::canonical(path).parent_path(); + auto opts = config_json::options_t{ + .expand_substitutions = resolveEnvVars, + .expand_includes = resolveInc, + .expand_deprecated = true, + .include_directories = std::span{&configPath, 1}, + }; + return config_json::load_config_file(path, opts).to_libjansson().release(); } - - return root; -} - -json_t *Config::load(const std::string &u, bool resolveInc, - bool resolveEnvVars) { - FILE *f; - - if (u == "-") - f = loadFromStdio(); - else - f = loadFromLocalFile(u); - - json_t *root = load(f, resolveInc, resolveEnvVars); - - fclose(f); - - return root; -} - -FILE *Config::loadFromStdio() { - logger->info("Reading configuration from standard input"); - - auto *cwd = new char[PATH_MAX]; - - configPath = getcwd(cwd, PATH_MAX); - - delete[] cwd; - - return stdin; -} - -FILE *Config::loadFromLocalFile(const std::string &u) { - logger->info("Reading configuration from local file: {}", u); - - configPath = u; - FILE *f = fopen(u.c_str(), "r"); - if (!f) - throw RuntimeError("Failed to open configuration from: {}", u); - - return f; -} - -json_t *Config::decode(FILE *f) { - json_error_t err; - - // Update list of include directories - auto incDirs = getIncludeDirectories(f); - includeDirectories.insert(includeDirectories.end(), incDirs.begin(), - incDirs.end()); - - json_t *root = json_loadf(f, 0, &err); - if (root == nullptr) { -#ifdef WITH_CONFIG - // We try again to parse the config in the legacy format - root = libconfigDecode(f); -#else - throw JanssonParseError(err); -#endif // WITH_CONFIG - } - - return root; -} - -std::list Config::getIncludeDirectories(FILE *f) const { - int ret, fd; - char buf[PATH_MAX]; - char *dir; - - std::list dirs; - - // Adding directory of base configuration file - fd = fileno(f); - if (fd < 0) - throw SystemError("Failed to get file descriptor"); - - auto path = fmt::format("/proc/self/fd/{}", fd); - - ret = readlink(path.c_str(), buf, sizeof(buf)); - if (ret > 0) { - buf[ret] = 0; - if (isLocalFile(buf)) { - dir = dirname(buf); - dirs.push_back(dir); - } - } - - // Adding current working directory - dir = getcwd(buf, sizeof(buf)); - if (dir != nullptr) - dirs.push_back(dir); - - return dirs; -} - -std::list Config::resolveIncludes(const std::string &n) { - glob_t gb; - int ret, flags = 0; - - memset(&gb, 0, sizeof(gb)); - - auto name = n; - resolveEnvVars(name); - - if (name.size() >= 1 && name[0] == '/') { // absolute path - ret = glob(name.c_str(), flags, nullptr, &gb); - if (ret && ret != GLOB_NOMATCH) - gb.gl_pathc = 0; - } else { // relative path - for (auto &dir : includeDirectories) { - auto pattern = fmt::format("{}/{}", dir, name.c_str()); - - ret = glob(pattern.c_str(), flags, nullptr, &gb); - if (ret && ret != GLOB_NOMATCH) { - gb.gl_pathc = 0; - - goto out; - } - - flags |= GLOB_APPEND; - } - } - -out: - std::list files; - for (unsigned i = 0; i < gb.gl_pathc; i++) - files.push_back(gb.gl_pathv[i]); - - globfree(&gb); - - return files; -} - -void Config::resolveEnvVars(std::string &text) { - static const std::regex env_re{R"--(\$\{([^}]+)\})--"}; - - std::smatch match; - while (std::regex_search(text, match, env_re)) { - auto const from = match[0]; - auto const var_name = match[1].str(); - char *var_value = std::getenv(var_name.c_str()); - if (!var_value) - throw RuntimeError("Unresolved environment variable: {}", var_name); - - text.replace(from.first - text.begin(), from.second - from.first, - var_value); - - logger->debug("Replace env var {} in \"{}\" with value \"{}\"", var_name, - text, var_value); - } -} - -#ifdef WITH_CONFIG -#if (LIBCONFIG_VER_MAJOR > 1) || \ - ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) -const char **Config::includeFuncStub(config_t *cfg, const char *include_dir, - const char *path, const char **error) { - void *ctx = config_get_hook(cfg); - - return reinterpret_cast(ctx)->includeFunc(cfg, include_dir, path, - error); -} - -const char **Config::includeFunc(config_t *cfg, const char *include_dir, - const char *path, const char **error) { - auto paths = resolveIncludes(path); - - unsigned i = 0; - auto files = (const char **)malloc(sizeof(char **) * (paths.size() + 1)); - - for (auto &path : paths) - files[i++] = strdup(path.c_str()); - - files[i] = NULL; - - return files; -} -#endif - -json_t *Config::libconfigDecode(FILE *f) { - int ret; - - config_t cfg; - config_setting_t *cfg_root; - config_init(&cfg); - config_set_auto_convert(&cfg, 1); - - // Setup libconfig include path -#if (LIBCONFIG_VER_MAJOR > 1) || \ - ((LIBCONFIG_VER_MAJOR == 1) && (LIBCONFIG_VER_MINOR >= 7)) - config_set_hook(&cfg, this); - - config_set_include_func(&cfg, includeFuncStub); -#else - if (includeDirectories.size() > 0) { - logger->info("Setting include dir to: {}", includeDirectories.front()); - - config_set_include_dir(&cfg, includeDirectories.front().c_str()); - - if (includeDirectories.size() > 1) { - logger->warn( - "Ignoring all but the first include directories for libconfig"); - logger->warn( - " libconfig does not support more than a single include dir!"); - } - } -#endif - - // Rewind before re-reading - rewind(f); - - ret = config_read(&cfg, f); - if (ret != CONFIG_TRUE) - throw LibconfigParseError(&cfg); - - cfg_root = config_root_setting(&cfg); - - json_t *root = config_to_json(cfg_root); - if (!root) - throw RuntimeError("Failed to convert JSON to configuration file"); - - config_destroy(&cfg); - - return root; -} -#endif // WITH_CONFIG - -json_t *Config::walkStrings(json_t *root, str_walk_fcn_t cb) { - const char *key; - size_t index; - json_t *val, *new_val, *new_root; - - switch (json_typeof(root)) { - case JSON_STRING: - return cb(root); - - case JSON_OBJECT: - new_root = json_object(); - - json_object_foreach (root, key, val) { - new_val = walkStrings(val, cb); - - json_object_set_new(new_root, key, new_val); - } - - return new_root; - - case JSON_ARRAY: - new_root = json_array(); - - json_array_foreach (root, index, val) { - new_val = walkStrings(val, cb); - - json_array_append_new(new_root, new_val); - } - - return new_root; - - default: - return json_incref(root); - }; -} - -json_t *Config::expandEnvVars(json_t *in) { - return walkStrings(in, [this](json_t *str) -> json_t * { - std::string text = json_string_value(str); - - resolveEnvVars(text); - - return json_string(text.c_str()); - }); -} - -json_t *Config::expandIncludes(json_t *in) { - return walkStrings(in, [this](json_t *str) -> json_t * { - int ret; - std::string text = json_string_value(str); - static const std::string kw = "@include "; - - auto res = std::mismatch(kw.begin(), kw.end(), text.begin()); - if (res.first != kw.end()) - return json_incref(str); - else { - std::string pattern = text.substr(kw.size()); - - resolveEnvVars(pattern); - - json_t *incl = nullptr; - - for (auto &path : resolveIncludes(pattern)) { - json_t *other = load(path); - if (!other) - throw ConfigError(str, "include", - "Failed to include config file from {}", path); - - if (!incl) - incl = other; - else if (json_is_object(incl) && json_is_object(other)) { - ret = json_object_update_recursive(incl, other); - if (ret) - throw ConfigError( - str, "include", - "Can not mix object and array-typed include files"); - } else if (json_is_array(incl) && json_is_array(other)) { - ret = json_array_extend(incl, other); - if (ret) - throw ConfigError( - str, "include", - "Can not mix object and array-typed include files"); - } - - logger->debug("Included config from: {}", path); - } - - return incl; - } - }); } diff --git a/lib/json.cpp b/lib/json.cpp new file mode 100644 index 000000000..585cb9210 --- /dev/null +++ b/lib/json.cpp @@ -0,0 +1,525 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +extern "C" { +#include +#include +#include +} + +#include +#include +#include + +using namespace std::string_view_literals; + +namespace villas { + +void libjansson_deleter::operator()(::json_t *json) const { json_decref(json); } + +config_json config_json_base::from_libjansson(::json_t const *json) { + switch (json_typeof(json)) { + using enum json_type; + + case JSON_ARRAY: { + std::size_t index; + ::json_t *value; + + auto array = config_json::array(); + json_array_foreach (json, index, value) + array.push_back(config_json::from_libjansson(value)); + + return array; + } + + case JSON_OBJECT: { + char const *key; + std::size_t keylen; + ::json_t *value; + + auto object = config_json::object(); + // The const_cast below is safe as long as we don't replace the contained value by + // accessing the underlying iterator with json_object_key_to_iter(key) and + // json_object_iter_set or json_object_iter_set_new. + // + // There is no API in libjansson to enumerate keys of a `::json_t const *` object. + // + // See https://github.com/akheron/jansson/issues/578 + json_object_keylen_foreach (const_cast<::json_t *>(json), key, keylen, + value) + object.emplace(std::string{key, keylen}, + config_json::from_libjansson(value)); + + return object; + } + + case JSON_STRING: { + return std::string{json_string_value(json), json_string_length(json)}; + } + + case JSON_INTEGER: { + return json_integer_value(json); + } + + case JSON_REAL: { + return json_real_value(json); + } + + case JSON_TRUE: { + return true; + } + + case JSON_FALSE: { + return false; + } + + case JSON_NULL: + default: { + return nullptr; + } + } +} + +libjansson_ptr config_json_base::to_libjansson() const { + // The static_cast below assumes that the *this object is actually a config_json + // object which is derived from config_json_base. This is ensured by the private + // constructor of `config_json_base` which makes config_json_base only constructible + // as part of a friend config_json instance. + // + // Read up on the CRTP C++ design pattern for more information. + auto const &self = *static_cast(this); + + switch (self.type()) { + using value_t = config_json::value_t; + + case value_t::array: { + auto array = libjansson_ptr(json_array()); + for (auto const &item : self) + json_array_append_new(array.get(), item.to_libjansson().release()); + + return array; + } + + case value_t::object: { + auto object = libjansson_ptr(json_object()); + for (auto const &[key, value] : self.items()) + json_object_setn_new_nocheck(object.get(), key.data(), key.size(), + value.to_libjansson().release()); + + return object; + } + + case value_t::string: { + auto const string = self.get(); + return libjansson_ptr(json_stringn_nocheck(string->data(), string->size())); + } + + case value_t::number_integer: { + auto const integer = self.get(); + return libjansson_ptr(json_integer(*integer)); + } + + case value_t::number_unsigned: { + auto const integer = self.get(); + return libjansson_ptr(json_integer(*integer)); + } + + case value_t::number_float: { + auto const real = self.get(); + return libjansson_ptr(json_real(*real)); + } + + case value_t::boolean: { + auto const boolean = self.get(); + return libjansson_ptr(json_boolean(*boolean)); + } + + case value_t::null: + return libjansson_ptr(json_null()); + + case value_t::binary: + throw std::invalid_argument{"cannot convert binary value to libjansson"}; + + case value_t::discarded: + throw std::invalid_argument{"cannot convert discarded value to libjansson"}; + + default: + __builtin_unreachable(); + } +} + +namespace { + +// implementation of deprecated variable substitutions +bool deprecatedSubstitution(config_json &value, + config_json::options_t options) { + if (not options.expand_deprecated or not value.is_string()) + return false; + + constexpr static auto COMPAT_INCLUDE_KEYWORD = "@include "sv; + auto logger = Log::get("config"); + auto string = value.get_ref(); + auto doInclude = false; + auto expanded = std::size_t{0}; + + // check for legacy @include keyword + if (options.expand_includes and string.starts_with(COMPAT_INCLUDE_KEYWORD)) { + doInclude = true; + expanded = COMPAT_INCLUDE_KEYWORD.length(); + } + + // legacy environment variable substitution syntax + static auto const envRegex = std::regex(R"--(\$\{([^}]+)\})--"); + enum : std::size_t { + CAPTURE_ALL = 0, + CAPTURE_NAME, + }; + + // expand legacy environment substition syntax + std::smatch match; + while (options.expand_substitutions and + std::regex_search(string.cbegin() + expanded, string.cend(), match, + envRegex)) { + auto name = std::string(match[CAPTURE_NAME]); + auto envPtr = std::getenv(name.c_str()); + if (not envPtr) + throw std::runtime_error( + fmt::format("Could substitute environment variable {:?}", name)); + + auto envValue = std::string_view(envPtr); + auto [begin, end] = std::pair(match[CAPTURE_ALL]); + string.replace(begin, end, envValue.begin(), envValue.end()); + expanded += match.position() + envValue.length(); + } + + // expand legacy @include directive + if (doInclude) { + auto pattern = + std::string_view(string).substr(COMPAT_INCLUDE_KEYWORD.length()); + auto result = config_json(nullptr); + for (auto path : utils::glob(pattern, options.include_directories)) { + auto partial_result = config_json::load_config_file(path, options); + if (result.is_null()) + result = partial_result; + else if (partial_result.is_object() and result.is_object()) + result.update(partial_result, true); + else if (partial_result.is_array() and result.is_array()) + result.insert(result.end(), partial_result.begin(), + partial_result.end()); + } + + logger->warn("Found deprecated @include directive: {}", value); + value = std::move(result); + } else if (expanded) { + logger->warn("Found deprecated environment substitution: {}", value); + value = std::move(string); + } + + return expanded != 0; +} + +void expandStringSubstitution(std::string &string) { + if (not string.starts_with("$"sv)) + return; + + if (string.starts_with("$$"sv)) { + string.erase(0); + return; + } + + constexpr static auto STRING_PREFIX = "$string:"sv; + if (string.starts_with(STRING_PREFIX)) { + auto name = string.c_str() + STRING_PREFIX.length(); + auto env = std::getenv(name); + if (not env) + throw std::runtime_error(fmt::format( + "Failed to substitute unknown environment variable {:?}", name)); + + string = env; + } + + throw std::runtime_error(fmt::format("Unknown substition type {:?}", string)); +} + +void expandValueSubstitution(config_json &value) { + if (not value.is_string()) + return; + + auto &string = value.get_ref(); + if (not string.starts_with("$"sv)) + return; + + if (string.starts_with("$$"sv)) { + string.erase(0); + return; + } + + constexpr static auto JSON_PREFIX = "$json:"sv; + if (string.starts_with(JSON_PREFIX)) { + auto name = string.c_str() + JSON_PREFIX.length(); + auto env = std::getenv(name); + if (not env) + throw std::runtime_error( + fmt::format("Failed to substitute environment variable {:?}", name)); + + value = config_json::parse(env); + return; + } + + constexpr static auto STRING_PREFIX = "$string:"sv; + if (string.starts_with(STRING_PREFIX)) { + auto name = string.c_str() + STRING_PREFIX.length(); + auto env = std::getenv(name); + if (not env) + throw std::runtime_error(fmt::format( + "Failed to substitute unknown environment variable {:?}", name)); + + string = env; + return; + } + + throw std::runtime_error(fmt::format("Unknown substition type {:?}", string)); +} + +config_json parseLibconfigSetting(::config_setting_t const *setting, + config_json::options_t opts) { + auto logger = Log::get("config"); + + switch (config_setting_type(setting)) { + case CONFIG_TYPE_ARRAY: + case CONFIG_TYPE_LIST: { + auto array = villas::config_json::array(); + for (auto const idx : std::views::iota(0, config_setting_length(setting))) { + auto const elem = config_setting_get_elem(setting, idx); + array.push_back(parseLibconfigSetting(elem, opts)); + } + + return array; + } + + case CONFIG_TYPE_GROUP: { + auto object = villas::config_json::object(); + for (auto const idx : std::views::iota(0, config_setting_length(setting))) { + auto const elem = config_setting_get_elem(setting, idx); + auto name = std::string(config_setting_name(elem)); + if (opts.expand_substitutions) + expandStringSubstitution(name); + object[std::move(name)] = parseLibconfigSetting(elem, opts); + } + + return object; + } + + case CONFIG_TYPE_STRING: { + auto json = config_json(std::string(config_setting_get_string(setting))); + if (not deprecatedSubstitution(json, opts) and opts.expand_substitutions) + expandValueSubstitution(json); + return json; + } + + case CONFIG_TYPE_INT: { + return config_setting_get_int(setting); + } + + case CONFIG_TYPE_INT64: { + return config_setting_get_int64(setting); + } + + case CONFIG_TYPE_FLOAT: { + return config_setting_get_float(setting); + } + + case CONFIG_TYPE_BOOL: { + return config_setting_get_bool(setting); + } + + case CONFIG_TYPE_NONE: + default: { + return nullptr; + } + } +} + +extern "C" char const **libconfigIncludeFunc(::config_t *config, char const *, + char const *pattern, + char const **error) noexcept { + auto opts = static_cast(config_get_hook(config)); + auto paths = std::vector{}; + + if (not opts->expand_includes) { + *error = "include directives are disabled"; + return nullptr; + } + + auto pattern_json = config_json(pattern); + deprecatedSubstitution(pattern_json, + { + .expand_substitutions = opts->expand_substitutions, + .expand_deprecated = opts->expand_deprecated, + }); + auto const &pattern_expanded = pattern_json.get_ref(); + + try { + paths = utils::glob(pattern_expanded, opts->include_directories); + std::erase_if(paths, [](auto const &path) { + auto discardErrorCode = std::error_code{}; + return not fs::is_regular_file(path, discardErrorCode); + }); + } catch (...) { + } + + if (paths.empty()) { + *error = "include directive did not match any file"; + return nullptr; + } + + auto ret = + static_cast(std::calloc(paths.size() + 1, sizeof(char *))); + auto index = std::size_t{0}; + for (auto &path : paths) + ret[index++] = strdup(path.c_str()); + + return ret; +} + +config_json readLibconfigFile(std::FILE *file, config_json::options_t opts) { + using ConfigDeleter = decltype([](::config_t *c) { ::config_destroy(c); }); + using ConfigPtr = std::unique_ptr<::config_t, ConfigDeleter>; + + ::config_t config; + ::config_init(&config); + auto guard = ConfigPtr(&config); + + ::config_set_hook(&config, &opts); + ::config_set_include_func(&config, &libconfigIncludeFunc); + if (auto ret = ::config_read(&config, file); ret != CONFIG_TRUE) { + throw std::runtime_error(fmt::format("Failed to load libconfig file: {}", + config_error_text(&config))); + } + + return parseLibconfigSetting(config_root_setting(&config), opts); +} + +class ConfigParser { +public: + ConfigParser(config_json::options_t const &opts = {}) : options(opts) {} + + bool operator()(int depth, config_json::parse_event_t event, + config_json &parsed) { + switch (event) { + case config_json::parse_event_t::key: { + return parseKey(parsed); + } + + case config_json::parse_event_t::value: { + return parseValue(depth, parsed); + } + + case config_json::parse_event_t::object_end: { + return finishObject(depth, parsed); + } + + default: + return true; + } + } + +private: + struct Include { + int depth; + config_json value; + }; + + constexpr static auto INCLUDE_KEYWORD = "$include"sv; + + bool parseKey(config_json &key) { + auto &keyString = key.get_ref(); + + valueIsIncludePattern = (keyString == INCLUDE_KEYWORD); + if (not valueIsIncludePattern and options.expand_substitutions) + expandStringSubstitution(keyString); + + return true; + } + + bool parseValue(int depth, config_json &value) { + // TODO: remove deprecated substitions + if (not deprecatedSubstitution(value, options) and + options.expand_substitutions) + expandValueSubstitution(value); + + if (not valueIsIncludePattern) { + return true; + } + + auto &pattern = value.get_ref(); + if (not options.expand_includes) + throw std::runtime_error{fmt::format( + "Failed to include {}: include directives are disabled", pattern)}; + + for (auto path : utils::glob(pattern, options.include_directories)) { + [[maybe_unused]] auto discardErrorCode = std::error_code{}; + if (not fs::is_regular_file(path, discardErrorCode)) + continue; + + auto includedValue = config_json::load_config_file(path, options); + includes.push_back({depth, includedValue}); + } + + return false; + } + + bool finishObject(int depth, config_json &object) { + if (options.expand_includes) { + while (not includes.empty() and includes.back().depth == depth + 1) { + object.update(includes.back().value, true); + includes.pop_back(); + } + + object.erase(INCLUDE_KEYWORD); + } + + return true; + } + + bool valueIsIncludePattern = false; + std::vector includes = {}; + config_json::options_t options; +}; + +} // namespace + +config_json config_json_base::load_config(std::FILE *f, options_t opts) { + return config_json::parse(f, ConfigParser{opts}); +} + +config_json config_json_base::load_config(std::string_view s, options_t opts) { + return config_json::parse(s.begin(), s.end(), ConfigParser{opts}); +} + +config_json config_json_base::load_config_file(fs::path const &path, + options_t opts) { + auto file = std::fopen(path.c_str(), "r"); + if (not file) { + throw std::runtime_error( + fmt::format("Failed to open config file {}", path.string())); + } + + if (auto ext = path.extension(); ext == ".json") { + return config_json::load_config(file, opts); + } else if (ext == ".conf") { + return readLibconfigFile(file, opts); + } else { + throw std::runtime_error(fmt::format( + "Failed to load config file with unknown extension {}", ext.string())); + } +} + +} // namespace villas diff --git a/lib/nodes/fpga.cpp b/lib/nodes/fpga.cpp index 45f1e3f24..167e4e072 100644 --- a/lib/nodes/fpga.cpp +++ b/lib/nodes/fpga.cpp @@ -379,8 +379,7 @@ int FpgaNodeFactory::start(SuperNode *sn) { } if (cards.empty()) { - auto searchPath = - sn->getConfigPath().substr(0, sn->getConfigPath().rfind("/")); + auto searchPath = sn->getConfigPath(); createCards(sn->getConfig(), cards, searchPath, vfioContainer); } diff --git a/packaging/nix/villas.nix b/packaging/nix/villas.nix index 246150a67..51d3e982b 100644 --- a/packaging/nix/villas.nix +++ b/packaging/nix/villas.nix @@ -1,9 +1,13 @@ # SPDX-FileCopyrightText: 2023 OPAL-RT Germany GmbH # SPDX-License-Identifier: Apache-2.0 { + lib, + stdenv, + makeWrapper, # General configuration src, version, + system, withGpl ? true, withAllExtras ? false, withAllFormats ? false, @@ -41,23 +45,24 @@ bash, cmake, coreutils, + curl, + gnugrep, graphviz, + jansson, jq, - lib, - makeWrapper, + libuuid, + libwebsockets, + nlohmann_json, + openssl, pkg-config, - stdenv, - system, + spdlog, # Optional dependencies boxfort, comedilib, criterion, - curl, czmq, cyrus_sasl, ethercat, - gnugrep, - jansson, lib60870, libconfig, libdatachannel, @@ -70,14 +75,11 @@ libsodium, libuldaq, libusb1, - libuuid, - libwebsockets, libxml2, lua, mosquitto, nanomsg, opendssc, - openssl, orchestra, pcre2, pkgsBuildBuild, @@ -89,7 +91,6 @@ rdkafka, rdma-core, redis-plus-plus, - spdlog, linuxHeaders, }: stdenv.mkDerivation { @@ -190,8 +191,9 @@ stdenv.mkDerivation { ]; propagatedBuildInputs = [ - libuuid jansson + libuuid + nlohmann_json ] ++ lib.optionals withFormatProtobuf [ protobuf diff --git a/tests/unit/config.cpp b/tests/unit/config.cpp index 0ecd3a2fe..7389a80e5 100644 --- a/tests/unit/config.cpp +++ b/tests/unit/config.cpp @@ -7,70 +7,88 @@ #include #include +#include #include +#include #include #include +using namespace std::string_view_literals; using namespace villas::node; +using FileDeleter = decltype([](std::FILE *f) { std::fclose(f); }); +using FilePtr = std::unique_ptr; + +constexpr auto fileNameTemplate = "villas.unit-test.XXXXXX.conf"sv; +constexpr auto fileNameSuffix = ".conf"sv; // cppcheck-suppress syntaxError Test(config, env) { - const char *cfg_f = "test = \"${MY_ENV_VAR}\"\n"; - - std::FILE *f = std::tmpfile(); - std::fputs(cfg_f, f); - std::rewind(f); - - auto c = Config(); - - char env[] = "MY_ENV_VAR=mobydick"; - putenv(env); - - auto *r = c.load(f); - cr_assert_not_null(r); - - auto *j = json_object_get(r, "test"); - cr_assert_not_null(j); - - cr_assert(json_is_string(j)); - cr_assert_str_eq("mobydick", json_string_value(j)); + auto configString = R"libconfig( + test = "${MY_ENV_VAR}" + )libconfig"; + + auto configPathTemplate = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto configFd = + ::mkstemps(configPathTemplate.data(), fileNameSuffix.length()); + auto configFile = FilePtr(::fdopen(configFd, "w")); + auto configPath = fs::path(configPathTemplate); + std::fputs(configString, configFile.get()); + configFile.reset(); + + auto config = Config(); + ::setenv("MY_ENV_VAR", "mobydick", true); + + auto *root = config.load(fs::path(configPath)); + cr_assert_not_null(root); + + auto *string = json_object_get(root, "test"); + cr_assert_not_null(string); + cr_assert(json_is_string(string)); + cr_assert_str_eq("mobydick", json_string_value(string)); } Test(config, include) { - const char *cfg_f2 = "magic = 1234\n"; - - char f2_fn_tpl[] = "/tmp/villas.unit-test.XXXXXX"; - int f2_fd = mkstemp(f2_fn_tpl); - - std::FILE *f2 = fdopen(f2_fd, "w"); - std::fputs(cfg_f2, f2); - std::rewind(f2); - - auto cfg_f1 = fmt::format("subval = \"@include {}\"\n", f2_fn_tpl); - - std::FILE *f1 = std::tmpfile(); - std::fputs(cfg_f1.c_str(), f1); - std::rewind(f1); - - auto env = fmt::format("{}", f2_fn_tpl); - setenv("INCLUDE_FILE", env.c_str(), true); - - auto c = Config(); - - auto *r = c.load(f1); - cr_assert_not_null(r); - - auto *j = json_object_get(r, "subval"); - cr_assert_not_null(j); - - auto *j2 = json_object_get(j, "magic"); - cr_assert_not_null(j2); - - cr_assert(json_is_integer(j2)); - cr_assert_eq(json_number_value(j2), 1234); - - std::fclose(f2); - std::remove(f2_fn_tpl); + auto incString = R"libconfig( + magic = 1234 + )libconfig"; + + auto incPathTemplate = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto incFd = ::mkstemps(incPathTemplate.data(), fileNameSuffix.length()); + cr_assert(incFd >= 0); + auto incFile = FilePtr(::fdopen(incFd, "w")); + cr_assert_not_null(incFile); + auto incPath = fs::path(incPathTemplate); + std::fputs(incString, incFile.get()); + incFile.reset(); + + auto configString = fmt::format(R"libconfig( + subval = {{ @include "{}" }} + )libconfig", + incPath.string()); + auto configPathTemplate = + std::string(fs::temp_directory_path() / fileNameTemplate); + auto configFd = + ::mkstemps(configPathTemplate.data(), fileNameSuffix.length()); + cr_assert(configFd >= 0); + auto configFile = FilePtr(::fdopen(configFd, "w")); + cr_assert_not_null(configFile); + auto configPath = fs::path(configPathTemplate); + std::fputs(configString.c_str(), configFile.get()); + configFile.reset(); + + auto config = Config(); + auto *root = config.load(configPath); + cr_assert_not_null(root); + + auto *subval = json_object_get(root, "subval"); + cr_assert_not_null(subval); + + auto *magic = json_object_get(subval, "magic"); + cr_assert_not_null(magic); + cr_assert(json_is_integer(magic)); + cr_assert_eq(json_number_value(magic), 1234); }