Merge pull request #15114 from omoerbeek/rec-rust-web

rec: move to embedded web service written in Rust

.github/actions/spell-check/expect.txt

@@ -917,7 +917,7 @@ Novell
 nproxy
 NPTL
 NSes
-nsid
+NSID
 nsis
 nsrecord
 nsset
@@ -1016,9 +1016,9 @@ pickchashed
 pickclosest
 pickhashed
 picknamehashed
-pickselfweighted
 pickrandom
 pickrandomsample
+pickselfweighted
 pickwhashed
 pickwrandom
 piddir
@@ -1498,7 +1498,7 @@ versionmodified
 Viala
 viewcode
 visitedlinkcolor
-vixie
+Vixie
 vla
 Voegeli
 Volker

DEVELOPMENT.md

@@ -46,7 +46,7 @@ make -nwk  | /path/to/compiledb -o- > compile_commands.json
 to generate the compilation database.
 For the authoritative server, the configure command is run in the top level directory, while the compiledb command should be run in the `pdns` subdirectory.
 
-# Seting up the LSP client
+# Setting up the LSP client
 
 Once the compilation database is generated, you can now move onto setting up an LSP client in your editor or IDE.
 

ext/yahttp/yahttp/meson.build

@@ -20,3 +20,7 @@ dep_yahttp = declare_dependency(
   link_with: lib_yahttp,
   include_directories: include_directories('..'),
 )
+
+dep_yahttp_header_only = declare_dependency(
+  include_directories: include_directories('..'),
+)

pdns/iputils.hh

@@ -27,7 +27,6 @@
 #include <iostream>
 #include <cstdio>
 #include <functional>
-#include <bitset>
 #include "pdnsexception.hh"
 #include "misc.hh"
 #include <netdb.h>

pdns/recursordist/Makefile.am

@@ -1,16 +1,16 @@
 JSON11_LIBS = $(top_builddir)/ext/json11/libjson11.la
 PROBDS_LIBS = $(top_builddir)/ext/probds/libprobds.la
 ARC4RANDOM_LIBS = $(top_builddir)/ext/arc4random/libarc4random.la
-RUST_LIBS = $(top_builddir)/settings/rust/libsettings.a $(LIBDL)
+RUST_LIBS = $(top_builddir)/rec-rust-lib/rust/librecrust.a $(LIBDL)
 
 AM_CPPFLAGS = $(LUA_CFLAGS) $(YAHTTP_CFLAGS) $(BOOST_CPPFLAGS) $(LIBSODIUM_CFLAGS) $(NET_SNMP_CFLAGS) $(LIBCAP_CFLAGS) $(SANITIZER_FLAGS) -O3 -Wall -pthread -DSYSCONFDIR=\"${sysconfdir}\" $(SYSTEMD_CFLAGS)
 
 AM_CPPFLAGS += \
 	-I$(top_srcdir)/ext/json11 \
 	-I$(top_srcdir)/ext/protozero/include \
-	-I$(top_srcdir)/settings \
-	-I$(top_builddir)/settings \
-	-I$(top_srcdir)/settings/rust/src \
+	-I$(top_srcdir)/rec-rust-lib \
+	-I$(top_builddir)/rec-rust-lib \
+	-I$(top_srcdir)/rec-rust-lib/rust/src \
 	$(YAHTTP_CFLAGS) \
 	$(LIBCRYPTO_INCLUDES) \
 	-DBOOST_CONTAINER_USE_STD_EXCEPTIONS
@@ -53,8 +53,8 @@ htmlfiles.h: incfiles ${srcdir}/html/* ${srcdir}/html/js/*
 rec-metrics-gen%h rec-prometheus-gen%h rec-snmp-gen%h rec-oids-gen%h RECURSOR-MIB%txt: metrics.py metrics_table.py RECURSOR-MIB.in
 	$(PYTHON) metrics.py
 
-# We explicitly build settings in two steps, as settings modifies files in the settings/rust subdir
-SUBDIRS=ext settings settings/rust
+# We explicitly build rec-rust-lib in two steps, as it modifies files in the rec-rust-lib/rust subdir
+SUBDIRS=ext rec-rust-lib rec-rust-lib/rust
 
 if LUA
 AM_CPPFLAGS +=$(LUA_CFLAGS)
@@ -189,6 +189,7 @@ pdns_recursor_SOURCES = \
 	rec-main.hh rec-main.cc \
 	rec-protozero.cc rec-protozero.hh \
 	rec-responsestats.hh rec-responsestats.cc \
+	rec-rust-lib/cxxsupport.cc \
 	rec-snmp.hh rec-snmp.cc \
 	rec-system-resolve.hh rec-system-resolve.cc \
 	rec-taskqueue.cc rec-taskqueue.hh \
@@ -212,7 +213,6 @@ pdns_recursor_SOURCES = \
 	rpzloader.cc rpzloader.hh \
 	secpoll-recursor.cc secpoll-recursor.hh \
 	secpoll.cc secpoll.hh \
-	settings/cxxsupport.cc \
 	sha.hh \
 	sholder.hh \
 	shuffle.cc shuffle.hh \
@@ -243,7 +243,7 @@ pdns_recursor_SOURCES = \
 	zoneparser-tng.cc zoneparser-tng.hh
 
 nodist_pdns_recursor_SOURCES = \
-	settings/cxxsettings-generated.cc
+	rec-rust-lib/cxxsettings-generated.cc
 
 if !HAVE_LUA_HPP
 BUILT_SOURCES += lua.hpp
@@ -253,7 +253,6 @@ endif
 CLEANFILES += lua.hpp
 
 pdns_recursor_LDADD = \
-	$(YAHTTP_LIBS) \
 	$(JSON11_LIBS) \
 	$(LIBCRYPTO_LIBS) \
 	$(BOOST_CONTEXT_LIBS) \
@@ -326,9 +325,11 @@ testrunner_SOURCES = \
 	rcpgenerator.cc \
 	rec-eventtrace.cc rec-eventtrace.hh \
 	rec-responsestats.hh rec-responsestats.cc \
+	rec-rust-lib/cxxsupport.cc \
 	rec-system-resolve.hh rec-system-resolve.cc \
 	rec-taskqueue.cc rec-taskqueue.hh \
 	rec-tcounters.cc rec-tcounters.hh \
+	rec-web-stubs.hh \
 	rec-xfrtracker.cc \
 	rec-zonetocache.cc rec-zonetocache.hh \
 	recpacketcache.cc recpacketcache.hh \
@@ -338,7 +339,6 @@ testrunner_SOURCES = \
 	root-dnssec.hh \
 	rpzloader.cc rpzloader.hh \
 	secpoll.cc \
-	settings/cxxsupport.cc \
 	sholder.hh \
 	shuffle.cc shuffle.hh \
 	sillyrecords.cc \
@@ -409,7 +409,7 @@ testrunner_SOURCES = \
 	zoneparser-tng.cc zoneparser-tng.hh
 
 nodist_testrunner_SOURCES = \
-	settings/cxxsettings-generated.cc
+	rec-rust-lib/cxxsettings-generated.cc
 
 testrunner_LDFLAGS = \
 	$(AM_LDFLAGS) \
@@ -542,18 +542,19 @@ rec_control_SOURCES = \
 	qtype.cc \
 	rcpgenerator.cc	rcpgenerator.hh \
 	rec-lua-conf.cc rec-lua-conf.hh \
+	rec-rust-lib/cxxsupport.cc \
 	rec-system-resolve.cc rec-system-resolve.hh \
+	rec-web-stubs.hh \
 	rec_channel.cc rec_channel.hh \
 	rec_control.cc \
-	settings/cxxsupport.cc \
 	sillyrecords.cc \
 	sortlist.cc sortlist.hh \
 	svc-records.cc svc-records.hh \
 	threadname.cc threadname.hh \
 	unix_utility.cc
 
 nodist_rec_control_SOURCES = \
-	settings/cxxsettings-generated.cc
+	rec-rust-lib/cxxsettings-generated.cc
 
 dnslabeltext.cc: dnslabeltext.rl
 	$(AM_V_GEN)$(RAGEL) $< -o dnslabeltext.cc

pdns/recursordist/configure.ac

@@ -192,8 +192,8 @@ AC_CONFIG_FILES([Makefile
 	ext/probds/Makefile
 	ext/yahttp/Makefile
 	ext/yahttp/yahttp/Makefile
-	settings/Makefile
-	settings/rust/Makefile])
+	rec-rust-lib/Makefile
+	rec-rust-lib/rust/Makefile])
 
 AC_OUTPUT
 

pdns/recursordist/ext/Makefile.am

@@ -1,6 +1,5 @@
 SUBDIRS = \
 	arc4random \
-	yahttp \
 	json11 \
 	probds
 

pdns/recursordist/meson.build

@@ -74,7 +74,7 @@ subdir('meson' / 'libresolv')               # res_query
 subdir('meson' / 'dnstap')                  # DNSTAP through libfstream
 subdir('meson' / 'libcurl')                 # Curl
 
-subdir('settings')
+subdir('rec-rust-lib')
 
 common_sources = []
 
@@ -300,7 +300,7 @@ dep_metrics = declare_dependency(
 deps = [
   dep_pdns,
   dep_no_config_in_source,
-  dep_rust_settings,
+  dep_rust_recrust,
   dep_boost,
   dep_boost_context,
   dep_threads,
@@ -313,7 +313,7 @@ deps = [
   dep_libssl,
   dep_lua,
   dep_protozero,
-  dep_yahttp,
+  dep_yahttp_header_only,
   dep_htmlfiles,
   dep_dnstap,
   dep_libcurl,
@@ -398,7 +398,7 @@ librec_common = declare_dependency(
     config_h,
     dependencies: [
       deps,
-      dep_settings_ch,
+      dep_recrust_ch,
       librec_dnslabeltext,
     ],
   )
@@ -423,10 +423,10 @@ tools = {
       dep_nod,
       dep_lua,
       dep_protozero,
-      dep_yahttp,
+      dep_yahttp_header_only,
       dep_json11,
-      dep_settings,
-      dep_rust_settings,
+      dep_recrust,
+      dep_rust_recrust,
       dep_systemd,
       librec_signers_openssl,
       librec_signers_sodium,
@@ -438,8 +438,8 @@ tools = {
     'manpages': ['rec_control.1'],
     'deps-extra': [
       dep_boost,
-      dep_settings,
-      dep_rust_settings,
+      dep_recrust,
+      dep_rust_recrust,
     ],
   },
 }
@@ -512,8 +512,8 @@ if get_option('unit-tests')
           dep_boost_test,
           dep_lua,
           dep_nod,
-          dep_settings,
-          dep_rust_settings,
+          dep_recrust,
+          dep_rust_recrust,
           librec_signers_openssl,
           librec_signers_sodium,
       ],

pdns/recursordist/rec-main.cc

@@ -39,11 +39,12 @@
 #include "secpoll-recursor.hh"
 #include "logging.hh"
 #include "dnsseckeeper.hh"
-#include "settings/cxxsettings.hh"
+#include "rec-rust-lib/cxxsettings.hh"
 #include "json.hh"
 #include "rec-system-resolve.hh"
 #include "root-dnssec.hh"
 #include "ratelimitedlog.hh"
+#include "rec-rust-lib/rust/web.rs.h"
 
 #ifdef NOD_ENABLED
 #include "nod.hh"
@@ -110,10 +111,10 @@ std::set<ComboAddress> g_proxyProtocolExceptions;
 boost::optional<ComboAddress> g_dns64Prefix{boost::none};
 DNSName g_dns64PrefixReverse;
 unsigned int g_maxChainLength;
-std::shared_ptr<SyncRes::domainmap_t> g_initialDomainMap; // new threads needs this to be setup
-std::shared_ptr<NetmaskGroup> g_initialAllowFrom; // new thread needs to be setup with this
-std::shared_ptr<NetmaskGroup> g_initialAllowNotifyFrom; // new threads need this to be setup
-std::shared_ptr<notifyset_t> g_initialAllowNotifyFor; // new threads need this to be setup
+LockGuarded<std::shared_ptr<SyncRes::domainmap_t>> g_initialDomainMap; // new threads needs this to be setup
+LockGuarded<std::shared_ptr<NetmaskGroup>> g_initialAllowFrom; // new thread needs to be setup with this
+LockGuarded<std::shared_ptr<NetmaskGroup>> g_initialAllowNotifyFrom; // new threads need this to be setup
+LockGuarded<std::shared_ptr<notifyset_t>> g_initialAllowNotifyFor; // new threads need this to be setup
 bool g_logRPZChanges{false};
 static time_t s_statisticsInterval;
 static std::atomic<uint32_t> s_counter;
@@ -272,6 +273,10 @@ int RecThreadInfo::runThreads(Logr::log_t log)
       taskInfo.start(currentThreadId, "task", cpusMap, log);
     }
 
+    if (::arg().mustDo("webserver")) {
+      serveRustWeb();
+    }
+
     currentThreadId = 1;
     auto& info = RecThreadInfo::info(currentThreadId);
     info.setListener();
@@ -349,6 +354,9 @@ int RecThreadInfo::runThreads(Logr::log_t log)
     info.setHandler();
     info.start(currentThreadId, "web+stat", cpusMap, log);
 
+    if (::arg().mustDo("webserver")) {
+      serveRustWeb();
+    }
     for (auto& tInfo : RecThreadInfo::infos()) {
       if (tInfo.getName() == "web+stat") { // XXX testing for isHandler() does not work as expected!
         continue;
@@ -1468,13 +1476,13 @@ void parseACLs()
     allowFrom = nullptr;
   }
 
-  g_initialAllowFrom = allowFrom;
+  *g_initialAllowFrom.lock() = allowFrom;
   // coverity[copy_constructor_call] maybe this can be avoided, but be careful as pointers get passed to other threads
   broadcastFunction([=] { return pleaseSupplantAllowFrom(allowFrom); });
 
   auto allowNotifyFrom = parseACL("allow-notify-from-file", "allow-notify-from", log);
 
-  g_initialAllowNotifyFrom = allowNotifyFrom;
+  *g_initialAllowNotifyFrom.lock() = allowNotifyFrom;
   // coverity[copy_constructor_call] maybe this can be avoided, but be careful as pointers get passed to other threads
   broadcastFunction([=] { return pleaseSupplantAllowNotifyFrom(allowNotifyFrom); });
 
@@ -2225,7 +2233,7 @@ static int serviceMain(Logr::log_t log)
   }
   g_networkTimeoutMsec = ::arg().asNum("network-timeout");
 
-  std::tie(g_initialDomainMap, g_initialAllowNotifyFor) = parseZoneConfiguration(g_yamlSettings);
+  std::tie(*g_initialDomainMap.lock(), *g_initialAllowNotifyFor.lock()) = parseZoneConfiguration(g_yamlSettings);
 
   g_latencyStatSize = ::arg().asNum("latency-statistic-size");
 
@@ -2822,10 +2830,10 @@ static void recursorThread()
     auto& threadInfo = RecThreadInfo::self();
     {
       SyncRes tmp(g_now); // make sure it allocates tsstorage before we do anything, like primeHints or so..
-      SyncRes::setDomainMap(g_initialDomainMap);
-      t_allowFrom = g_initialAllowFrom;
-      t_allowNotifyFrom = g_initialAllowNotifyFrom;
-      t_allowNotifyFor = g_initialAllowNotifyFor;
+      SyncRes::setDomainMap(*g_initialDomainMap.lock());
+      t_allowFrom = *g_initialAllowFrom.lock();
+      t_allowNotifyFrom = *g_initialAllowNotifyFrom.lock();
+      t_allowNotifyFor = *g_initialAllowNotifyFor.lock();
       t_udpclientsocks = std::make_unique<UDPClientSocks>();
       t_tcpClientCounts = std::make_unique<tcpClientCounts_t>();
       if (g_proxyMapping) {
@@ -2907,24 +2915,9 @@ static void recursorThread()
     }
 
     t_fdm = unique_ptr<FDMultiplexer>(getMultiplexer(log));
-
-    std::unique_ptr<RecursorWebServer> rws;
-
     t_fdm->addReadFD(threadInfo.getPipes().readToThread, handlePipeRequest);
 
     if (threadInfo.isHandler()) {
-      if (::arg().mustDo("webserver")) {
-        SLOG(g_log << Logger::Warning << "Enabling web server" << endl,
-             log->info(Logr::Info, "Enabling web server"));
-        try {
-          rws = make_unique<RecursorWebServer>(t_fdm.get());
-        }
-        catch (const PDNSException& e) {
-          SLOG(g_log << Logger::Error << "Unable to start the internal web server: " << e.reason << endl,
-               log->error(Logr::Critical, e.reason, "Exception while starting internal web server"));
-          _exit(99);
-        }
-      }
       SLOG(g_log << Logger::Info << "Enabled '" << t_fdm->getName() << "' multiplexer" << endl,
            log->info(Logr::Info, "Enabled multiplexer", "name", Logging::Loggable(t_fdm->getName())));
     }

pdns/recursordist/rec-main.hh

@@ -198,6 +198,7 @@ using RemoteLoggerStats_t = std::unordered_map<std::string, RemoteLoggerInterfac
 
 extern bool g_yamlSettings;
 extern string g_yamlSettingsSuffix;
+extern LockGuarded<pdns::rust::settings::rec::Recursorsettings> g_yamlStruct;
 extern bool g_logCommonErrors;
 extern size_t g_proxyProtocolMaximumSize;
 extern std::atomic<bool> g_quiet;
@@ -233,10 +234,10 @@ extern std::atomic<bool> g_statsWanted;
 extern uint32_t g_disthashseed;
 extern int g_argc;
 extern char** g_argv;
-extern std::shared_ptr<SyncRes::domainmap_t> g_initialDomainMap; // new threads needs this to be setup
-extern std::shared_ptr<NetmaskGroup> g_initialAllowFrom; // new thread needs to be setup with this
-extern std::shared_ptr<NetmaskGroup> g_initialAllowNotifyFrom; // new threads need this to be setup
-extern std::shared_ptr<notifyset_t> g_initialAllowNotifyFor; // new threads need this to be setup
+extern LockGuarded<std::shared_ptr<SyncRes::domainmap_t>> g_initialDomainMap; // new threads needs this to be setup
+extern LockGuarded<std::shared_ptr<NetmaskGroup>> g_initialAllowFrom; // new thread needs to be setup with this
+extern LockGuarded<std::shared_ptr<NetmaskGroup>> g_initialAllowNotifyFrom; // new threads need this to be setup
+extern LockGuarded<std::shared_ptr<notifyset_t>> g_initialAllowNotifyFor; // new threads need this to be setup
 extern thread_local std::shared_ptr<Regex> t_traceRegex;
 extern thread_local FDWrapper t_tracefd;
 extern string g_programname;