HPCC-33260 Allow a BM DFS service+dafilesrv to be secured

1) Ensure BM DFS meta is remapped to point to the BM dafilesrv
and correct secure port.
2) Allow dafilesrv to be configured with a CA/certs/trusted peers,
so that it can be configured to only accepts connections from
trusted clients.

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

dali/base/dautils.cpp

@@ -3675,6 +3675,11 @@ static CConfigUpdateHook directIOUpdateHook;
 static CriticalSection dafileSrvNodeCS;
 static Owned<INode> tlsDirectIONode, nonTlsDirectIONode;
 
+unsigned getPreferredDaFsServerPort()
+{
+    return getPreferredDafsClientPort(true);
+}
+
 void remapGroupsToDafilesrv(IPropertyTree *file, bool foreign, bool secure)
 {
     Owned<IPropertyTreeIterator> iter = file->getElements("Cluster");
@@ -3683,7 +3688,7 @@ void remapGroupsToDafilesrv(IPropertyTree *file, bool foreign, bool secure)
         IPropertyTree &cluster = iter->query();
         const char *planeName = cluster.queryProp("@name");
         Owned<IStoragePlane> plane = getDataStoragePlane(planeName, true);
-        if ((0 == plane->queryHosts().size()) && isAbsolutePath(plane->queryPrefix())) // if hosts group, or url, don't touch
+        if (isAbsolutePath(plane->queryPrefix())) // if url (i.e. not absolute prefix path) don't touch
         {
             if (isContainerized())
             {

dali/base/dautils.hpp

@@ -583,7 +583,7 @@ inline unsigned calcStripeNumber(unsigned partNum, const char *lfnName, unsigned
 }
 interface INamedGroupStore;
 extern da_decl void remapGroupsToDafilesrv(IPropertyTree *file, bool foreign, bool secure);
-
+extern da_decl unsigned getPreferredDaFsServerPort();
 #ifdef NULL_DALIUSER_STACKTRACE
 extern da_decl void logNullUser(IUserDescriptor *userDesc);
 #else

esp/services/ws_dfsservice/ws_dfsservice.cpp

@@ -126,6 +126,26 @@ static void populateLFNMeta(IUserDescriptor *userDesc, const char *logicalName,
         {
             VStringBuffer storagePlaneXPath("storage/%s", planeXPath.str());
             Owned<IPropertyTree> dataPlane = getGlobalConfigSP()->getPropTree(storagePlaneXPath);
+
+            const char *hostGroupName = dataPlane->queryProp("@hostGroup");
+            if (!isEmptyString(hostGroupName))
+            {
+                Owned<IPropertyTree> hostGroup = getHostGroup(hostGroupName, false);
+                if (hostGroup)
+                {
+                    // This is only likely to be used if this service is in BM
+                    // Cloud based storage planes are unlikely to be backed by hosts/hostGroups
+                    dataPlane.setown(createPTreeFromIPT(dataPlane));
+                    unsigned daFsSrvPort = getPreferredDaFsServerPort();
+                    Owned<IPropertyTreeIterator> iter = hostGroup->getElements("hosts");
+                    ForEach(*iter)
+                    {
+                        VStringBuffer endpoint("%s:%u", iter->query().queryProp(nullptr), daFsSrvPort);
+                        dataPlane->addProp("hosts", endpoint);
+                    }
+                    dataPlane->removeProp("@hostGroup");
+                }
+            }
             metaRoot->addPropTree("planes", dataPlane.getClear());
         }
     }
@@ -135,6 +155,8 @@ static void populateLFNMeta(IUserDescriptor *userDesc, const char *logicalName,
 void CWsDfsEx::init(IPropertyTree *cfg, const char *process, const char *service)
 {
     DBGLOG("Initializing %s service [process = %s]", service, process);
+    VStringBuffer xpath("Software/EspProcess/EspBinding[@service=\"%s\"]/@protocol", service);
+    isHttps = strsame("https", cfg->queryProp(xpath));
 }
 
 bool CWsDfsEx::onGetLease(IEspContext &context, IEspLeaseRequest &req, IEspLeaseResponse &resp)
@@ -177,8 +199,13 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r
         if (req.getAccessViaDafilesrv())
             opts |= LfnMOptRemap;
 
-        // NB: if we ever have some services with tls, and some without in bare-metal, this may need revisiting.
-        if (getComponentConfigSP()->getPropBool("@tls"))
+        if (isContainerized())
+        {
+            // NB: if we ever have some services with tls, and some without in bare-metal, this may need revisiting.
+            if (getComponentConfigSP()->getPropBool("@tls"))
+                opts |= LfnMOptTls;
+        }
+        else if (isHttps)
             opts |= LfnMOptTls;
 
         Owned<IPropertyTree> responseTree = createPTree();

esp/services/ws_dfsservice/ws_dfsservice.hpp

@@ -26,6 +26,7 @@
 
 class CWsDfsEx : public CWsDfs
 {
+    bool isHttps = false;
 public:
     virtual ~CWsDfsEx() {}
     virtual void init(IPropertyTree *cfg, const char *process, const char *service);

fs/dafilesrv/dafilesrv.cpp

@@ -585,11 +585,18 @@ int main(int argc, const char* argv[])
                 rowServiceConfiguration = daFileSrv->queryProp("@rowServiceConfiguration");
 
             // merge in bare-metal dafilesrv component expert settings
-            IPropertyTree *componentExpert = nullptr;
-            componentExpert = daFileSrv->queryPropTree("expert");
+            IPropertyTree *componentExpert = daFileSrv->queryPropTree("expert");
             if (componentExpert)
                 synchronizePTree(expert, componentExpert, false, true);
 
+            // merge in bare-metal dafilesrv component cert settings into newConfig
+            IPropertyTree *componentCert = daFileSrv->queryPropTree("cert");
+            if (componentCert)
+            {
+                IPropertyTree *cert = ensurePTree(newConfig, "cert");
+                synchronizePTree(cert, componentCert, false, true);
+            }
+
             // any overrides by Instance definitions?
             Owned<IPropertyTreeIterator> iter = daFileSrv->getElements("Instance");
             ForEach(*iter)

fs/dafsserver/dafsserver.cpp

@@ -134,18 +134,25 @@ static ISecureSocket *createSecureSocket(ISocket *sock, bool disableClientCertVe
         {
 #ifdef _CONTAINERIZED
             /* Connections are expected from 3rd parties via TLS,
-             * we do not expect them to provide a valid certificate for verification.
-             * Currently the server (this dafilesrv), will use either the "public" certificate issuer,
-             * unless it's visibility is "cluster" (meaning internal only)
-             */
+            * we do not expect them to provide a valid certificate for verification.
+            * Currently the server (this dafilesrv), will use either the "public" certificate issuer,
+            * unless it's visibility is "cluster" (meaning internal only)
+            */
 
             const char *certScope = strsame("cluster", getComponentConfigSP()->queryProp("service/@visibility")) ? "local" : "public";
             Owned<const ISyncedPropertyTree> info = getIssuerTlsSyncedConfig(certScope, nullptr, disableClientCertVerification);
             if (!info || !info->isValid())
                 throw makeStringException(-1, "createSecureSocket() : missing MTLS configuration");
             secureContextServer.setown(createSecureSocketContextSynced(info, ServerSocket));
 #else
-            secureContextServer.setown(createSecureSocketContextEx2(securitySettings.getSecureConfig(), ServerSocket));
+            Owned<IPropertyTree> cert = getComponentConfigSP()->getPropTree("cert");
+            if (cert)
+            {
+                Owned<ISyncedPropertyTree> certSyncedWrapper = createSyncedPropertyTree(cert);
+                secureContextServer.setown(createSecureSocketContextSynced(certSyncedWrapper, ServerSocket));
+            }
+            else
+                secureContextServer.setown(createSecureSocketContextEx2(securitySettings.getSecureConfig(), ServerSocket));
 #endif
         }
     }
@@ -5449,7 +5456,11 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
                     securitySettings.privateKey = nullptr;
                 }
             }
-            else
+            else if (!isContainerized() && getComponentConfigSP()->hasProp("cert"))
+            {
+                // validated when context is created in createSecureSocket
+            }
+            else // using environment.conf HPCCCertificateFile etc.
                 validateSSLSetup();
 #endif