Parse configuration files using nlohmann::json

CMakeLists.txt

@@ -74,6 +74,7 @@ endif()
 # Check packages
 find_package(PkgConfig REQUIRED)
 find_package(Threads REQUIRED)
+find_package(nlohmann_json REQUIRED)
 find_package(OpenMP)
 find_package(IBVerbs)
 find_package(RDMACM)

common/include/villas/utils.hpp

@@ -12,6 +12,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <list>
+#include <span>
 #include <string>
 #include <vector>
 
@@ -212,6 +213,10 @@ template <class... Ts> struct overloaded : Ts... {
 // Explicit deduction guide (not needed as of C++20)
 template <class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 
+// glob-style filesystem pattern matching
+std::vector<fs::path> glob(fs::path const &pattern,
+                           std::span<const fs::path> searchDirectories);
+
 void write_to_file(std::string data, const fs::path file);
 std::vector<std::string> read_names_in_directory(const fs::path &directory);
 

common/lib/utils.cpp

@@ -12,12 +12,14 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <string>
 #include <vector>
 
 #include <dirent.h>
 #include <fcntl.h>
+#include <fnmatch.h>
 #include <jansson.h>
 #include <openssl/bio.h>
 #include <openssl/buffer.h>
@@ -352,6 +354,70 @@ bool isPrivileged() {
   return true;
 }
 
+// internal glob implementation details
+namespace {
+bool isGlobPattern(fs::path const &path) {
+  static const auto specialCharacters = fs::path("?*[").native();
+  auto const &string = path.native();
+  return std::ranges::find_first_of(string, specialCharacters) != string.end();
+}
+
+bool isGlobMatch(fs::path const &pattern, fs::path const &path) {
+  return ::fnmatch(pattern.c_str(), path.c_str(), FNM_PATHNAME) == 0;
+}
+
+void globImpl(std::vector<fs::path> &result, fs::path &&path,
+              std::ranges::subrange<fs::path::iterator> pattern) {
+  [[maybe_unused]] auto discardErrorCode = std::error_code{};
+
+  if (pattern.empty()) {
+    // we've reached the end of our pattern
+    if (fs::exists(path, discardErrorCode))
+      result.push_back(path);
+    return;
+  }
+
+  if (not fs::is_directory(path, discardErrorCode))
+    return;
+
+  if (not isGlobPattern(pattern.front())) {
+    path /= pattern.front();
+    return globImpl(result, std::move(path), std::move(pattern).next());
+  } else {
+    auto nextPattern = pattern.next();
+    for (auto entry : fs::directory_iterator(path)) {
+      if (not isGlobMatch(pattern.front(), entry.path().filename()))
+        continue;
+
+      globImpl(result, fs::path(entry.path()), nextPattern);
+    }
+  }
+}
+} // namespace
+
+std::vector<fs::path> glob(fs::path const &pattern,
+                           std::span<const fs::path> searchDirectories) {
+  auto logger = Log::get("glob");
+  std::vector<fs::path> result;
+  if (pattern.is_absolute()) {
+    logger->debug("Matching absolute pattern {:?}", pattern.string());
+    globImpl(result, pattern.root_path(), pattern);
+  } else {
+    for (auto path : searchDirectories) {
+      logger->debug("Matching relative pattern {:?} in {:?}", pattern.string(),
+                    path.string());
+      globImpl(result, std::move(path), pattern);
+    }
+  }
+
+  if (result.empty()) {
+    throw std::runtime_error(
+        fmt::format("Could not find any file matching {:?}", pattern.string()));
+  }
+
+  return result;
+}
+
 void write_to_file(std::string data, const fs::path file) {
   villas::Log::get("Filewriter")->debug("{} > {}", data, file.string());
   std::ofstream outputFile(file.string());

include/villas/config_class.hpp

@@ -8,7 +8,6 @@
 #pragma once
 
 #include <cstdio>
-#include <functional>
 
 #include <jansson.h>
 #include <unistd.h>
@@ -25,71 +24,26 @@ namespace villas {
 namespace node {
 
 class Config {
-
-protected:
-  using str_walk_fcn_t = std::function<json_t *(json_t *)>;
-
+private:
   Logger logger;
-
-  std::list<std::string> includeDirectories;
-  std::string configPath;
-
-  // Check if file exists on local system.
-  static bool isLocalFile(const std::string &uri) {
-    return access(uri.c_str(), F_OK) != -1;
-  }
-
-  // Decode configuration file.
-  json_t *decode(FILE *f);
-
-#ifdef WITH_CONFIG
-  // Convert libconfig .conf file to libjansson .json file.
-  json_t *libconfigDecode(FILE *f);
-
-  static const char **includeFuncStub(config_t *cfg, const char *include_dir,
-                                      const char *path, const char **error);
-
-  const char **includeFunc(config_t *cfg, const char *include_dir,
-                           const char *path, const char **error);
-#endif // WITH_CONFIG
-
-  // Load configuration from standard input (stdim).
-  FILE *loadFromStdio();
-
-  // Load configuration from local file.
-  FILE *loadFromLocalFile(const std::string &u);
-
-  std::list<std::string> resolveIncludes(const std::string &name);
-
-  void resolveEnvVars(std::string &text);
-
-  // Resolve custom include directives.
-  json_t *expandIncludes(json_t *in);
-
-  // To shell-like subsitution of environment variables in strings.
-  json_t *expandEnvVars(json_t *in);
-
-  // Run a callback function for each string in the config
-  json_t *walkStrings(json_t *in, str_walk_fcn_t cb);
-
-  // Get the include dirs
-  std::list<std::string> getIncludeDirectories(FILE *f) const;
+  fs::path configPath;
 
 public:
   json_t *root;
 
   Config();
-  Config(const std::string &u);
+  Config(fs::path path);
 
+  Config(Config const &) = delete;
+  Config &operator=(Config const &) = delete;
+  Config(Config &&) = delete;
+  Config &operator=(Config &&) = delete;
   ~Config();
 
-  json_t *load(std::FILE *f, bool resolveIncludes = true,
-               bool resolveEnvVars = true);
-
-  json_t *load(const std::string &u, bool resolveIncludes = true,
+  json_t *load(fs::path path, bool resolveIncludes = true,
                bool resolveEnvVars = true);
 
-  std::string const &getConfigPath() const { return configPath; }
+  fs::path const &getConfigPath() const { return configPath; }
 };
 
 } // namespace node

include/villas/json.hpp

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <memory>
+#include <span>
+
+#include <fmt/ostream.h>
+#include <nlohmann/json.hpp>
+
+#include <villas/fs.hpp>
+#include <villas/json_fwd.hpp>
+
+// libjansson forward declaration
+struct json_t;
+
+namespace villas {
+
+// deleter for libjansson ::json_t * values
+struct libjansson_deleter {
+  void operator()(::json_t *) const;
+};
+
+// smart pointer for libjansson ::json_t * values.
+using libjansson_ptr = std::unique_ptr<::json_t, libjansson_deleter>;
+
+// base class which injects VILLASnode specific functionality
+// into the config_json specialization of nlohmann::basic_json.
+class config_json_base {
+private:
+  friend config_json;
+  config_json_base() = default;
+
+public:
+  // libjansson compatability
+  static config_json from_libjansson(::json_t const *);
+  libjansson_ptr to_libjansson() const;
+
+  // configuration parsing options
+  struct options_t {
+    // expand $ENV variable substitutions
+    bool expand_substitutions = false;
+    // expand $include keys
+    bool expand_includes = false;
+    // expand deprecated ${ENV} substitutions
+    bool expand_deprecated = false;
+    // a set of base directories to search for includes
+    std::span<const fs::path> include_directories = {};
+  };
+
+  // load a VILLASnode configuration
+  static config_json load_config(std::FILE *, options_t);
+  static config_json load_config(std::string_view, options_t);
+  static config_json load_config_file(fs::path const &, options_t);
+};
+
+}; // namespace villas
+
+template <> // format config_json using operator<<
+struct fmt::formatter<villas::config_json> : ostream_formatter {};
+
+template <> // format config_json_pointer using operator<<
+struct fmt::formatter<villas::config_json_pointer> : ostream_formatter {};

include/villas/json_fwd.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include <nlohmann/json_fwd.hpp>
+
+struct json_t;
+
+namespace villas {
+
+class config_json_base;
+
+using config_json =
+    nlohmann::basic_json<std::map,                  // ObjectType
+                         std::vector,               // ArrayType
+                         std::string,               // StringType
+                         bool,                      // BooleanType
+                         std::int64_t,              // NumberIntegerType
+                         std::uint64_t,             // NumberUnsignedType
+                         double,                    // NumberFloatType
+                         std::allocator,            // AllocatorType
+                         nlohmann::adl_serializer,  // JSONSerializer
+                         std::vector<std::uint8_t>, // BinaryType
+                         config_json_base           // CustomBaseClass
+                         >;
+
+using config_json_pointer = nlohmann::json_pointer<std::string>;
+
+}; // namespace villas

include/villas/super_node.hpp

@@ -140,7 +140,7 @@ class SuperNode {
 
   json_t *getConfig() { return config.root; }
 
-  const std::string &getConfigPath() const { return config.getConfigPath(); }
+  fs::path const &getConfigPath() const { return config.getConfigPath(); }
 
   int getAffinity() const { return affinity; }
 

lib/CMakeLists.txt

@@ -13,6 +13,7 @@ list(APPEND INCLUDE_DIRS
 
 set(LIBRARIES
     villas-common
+    nlohmann_json::nlohmann_json
     PkgConfig::JANSSON
     PkgConfig::UUID
     m
@@ -26,6 +27,7 @@ set(LIB_SRC
     config.cpp
     dumper.cpp
     format.cpp
+    json.cpp
     mapping.cpp
     mapping_list.cpp
     memory.cpp