Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

# Conflicts:
#	helm/hpcc/Chart.yaml
#	helm/hpcc/templates/_helpers.tpl
#	helm/hpcc/templates/dafilesrv.yaml
#	helm/hpcc/templates/dali.yaml
#	helm/hpcc/templates/dfuserver.yaml
#	helm/hpcc/templates/eclagent.yaml
#	helm/hpcc/templates/eclccserver.yaml
#	helm/hpcc/templates/eclscheduler.yaml
#	helm/hpcc/templates/esp.yaml
#	helm/hpcc/templates/localroxie.yaml
#	helm/hpcc/templates/roxie.yaml
#	helm/hpcc/templates/sasha.yaml
#	helm/hpcc/templates/thor.yaml
#	version.cmake

.github/workflows/build-assets.yml

@@ -185,7 +185,7 @@ jobs:
         run: |
           mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
-          plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM")
+          plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY")
           for plugin in "${plugins[@]}"; do
             sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
             sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
@@ -271,14 +271,17 @@ jobs:
         run: |
           mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
-          sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
-          sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-          docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \
-            cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \
-            cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
-          # Disabled as not currently needed ---
-          # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" 
-          # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package"
+          plugins=("PLATFORM" "CLIENTTOOLS_ONLY")
+          for plugin in "${plugins[@]}"; do
+            sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
+            sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
+            docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \
+              cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -D$plugin=ON -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \
+              cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
+            # Disabled as not currently needed ---
+            # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" 
+            # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package"
+          done
 
       - name: CMake Containerized Packages (internal)
         if: ${{ matrix.ln && matrix.container && !matrix.documentation }}
@@ -305,7 +308,7 @@ jobs:
           repo: LN
           token: ${{ secrets.LNB_TOKEN }}
           tag: ${{ needs.preamble.outputs.internal_tag }}
-          artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-internal*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-internal*.rpm"
+          artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-*-internal*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-*-internal*.rpm"
 
       - name: Locate k8s deb file (internal)
         if: ${{ matrix.ln && matrix.container && !matrix.documentation }}

common/thorhelper/roxiehelper.cpp

@@ -2837,7 +2837,7 @@ void loadHttpHeaders(IProperties *p, const char *finger)
 void HttpHelper::parseRequestHeaders(const char *headers)
 {
     if (!reqHeaders)
-        reqHeaders.setown(createProperties());
+        reqHeaders.setown(createProperties(true));
     loadHttpHeaders(reqHeaders, headers);
     const char *val = queryRequestHeader("Content-Type");
     if (val)

dali/base/dadfs.cpp

@@ -1181,7 +1181,6 @@ protected: friend class CDistributedFile;
     IDistributedSuperFile *lookupSuperFile(const char *logicalname, IUserDescriptor *user, AccessMode accessMode, IDistributedFileTransaction *transaction, unsigned timeout=INFINITE);
 
     SecAccessFlags getFilePermissions(const char *lname,IUserDescriptor *user,unsigned auditflags);
-    SecAccessFlags getNodePermissions(const IpAddress &ip,IUserDescriptor *user,unsigned auditflags);
     SecAccessFlags getFDescPermissions(IFileDescriptor *,IUserDescriptor *user,unsigned auditflags=0);
     SecAccessFlags getDLFNPermissions(CDfsLogicalFileName &dlfn,IUserDescriptor *user,unsigned auditflags=0);
     SecAccessFlags getDropZoneScopePermissions(const char *dropZoneName,const char *dropZonePath,IUserDescriptor *user,unsigned auditflags=0);
@@ -11808,16 +11807,6 @@ SecAccessFlags CDistributedFileDirectory::getFilePermissions(const char *lname,I
     return getDLFNPermissions(dlfn,user,auditflags);
 }
 
-SecAccessFlags CDistributedFileDirectory::getNodePermissions(const IpAddress &ip,IUserDescriptor *user,unsigned auditflags)
-{
-    if (ip.isNull())
-        return SecAccess_None;
-    CDfsLogicalFileName dlfn;
-    SocketEndpoint ep(0,ip);
-    dlfn.setExternal(ep,"/x");
-    return getDLFNPermissions(dlfn,user,auditflags);
-}
-
 SecAccessFlags CDistributedFileDirectory::getFDescPermissions(IFileDescriptor *fdesc,IUserDescriptor *user,unsigned auditflags)
 {
     // this checks have access to the nodes in the file descriptor

dali/base/dadfs.hpp

@@ -644,7 +644,6 @@ interface IDistributedFileDirectory: extends IInterface
     virtual SecAccessFlags getFilePermissions(const char *lname,IUserDescriptor *user,unsigned auditflags=0)=0; // see dasess for auditflags values
     virtual void setDefaultUser(IUserDescriptor *user)=0;
     virtual IUserDescriptor* queryDefaultUser()=0;
-    virtual SecAccessFlags getNodePermissions(const IpAddress &ip,IUserDescriptor *user,unsigned auditflags=0)=0;
     virtual SecAccessFlags getFDescPermissions(IFileDescriptor *,IUserDescriptor *user,unsigned auditflags=0)=0;
     virtual SecAccessFlags getDLFNPermissions(CDfsLogicalFileName &dlfn,IUserDescriptor *user,unsigned auditflags=0)=0;
     virtual SecAccessFlags getDropZoneScopePermissions(const char *dropZoneName,const char *dropZonePath,IUserDescriptor *user,unsigned auditflags=0)=0;

dali/base/dasds.cpp

@@ -5935,6 +5935,24 @@ IStoreHelper *createStoreHelper(const char *storeName, const char *location, con
 
 ///////////////
 
+static CConfigUpdateHook configUpdateHook;
+static void initializeStorageGroups(IPropertyTree *oldEnvironment) // oldEnvironment always null in containerized
+{
+    bool forceGroupUpdate = getComponentConfigSP()->getPropBool("DFS/@forceGroupUpdate");
+    initClusterAndStoragePlaneGroups(forceGroupUpdate, oldEnvironment);
+    if (isContainerized())
+    {
+        auto updateFunc = [](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration)
+        {
+            bool forceGroupUpdate = getComponentConfigSP()->getPropBool("DFS/@forceGroupUpdate");
+            initClusterAndStoragePlaneGroups(forceGroupUpdate, nullptr);
+        };
+        configUpdateHook.installOnce(updateFunc, false);
+    }
+}
+
+///////////////
+
 #ifdef _MSC_VER
 #pragma warning (push)
 #pragma warning (disable : 4355)  // 'this' : used in base member initializer list
@@ -6574,8 +6592,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort)
     Owned<IRemoteConnection> conn = connect("/", 0, RTM_INTERNAL, INFINITE);
     initializeInternals(conn->queryRoot());
     conn.clear();
-    bool forceGroupUpdate = config.getPropBool("DFS/@forceGroupUpdate");
-    initClusterAndStoragePlaneGroups(forceGroupUpdate, oldEnvironment);
+    initializeStorageGroups(oldEnvironment);
 }
 
 void CCovenSDSManager::saveStore(const char *storeName, bool currentEdition)

devdoc/roxie.md

@@ -281,3 +281,17 @@ Should the scope of the blacklist be different? Possible scopes are:
 
 Options 2 and 4 above would allow all aspects of the blacklisting behaviour to be specified by options on the SOAPCALL. We could control whether or not the
 blacklister is to be used at all via a SOAPCALL option with any of the above...
+
+Some notes on LocalAgent mode
+=============================
+
+In localAgent mode, the global queueManager object (normally a RoxieUdpSocketQueueManager) is replaced by a RoxieLocalQueueManager. Outbound packets are added directly to target queue, inbound are packed into DataBuffers.
+
+There is also "local optimizations" mode where any index operation reading a one-part file (does the same apply to one-part disk files?) just reads it directly on the server (regardless of localAgent setting). Typically still injected into receiver code though as otherwise handling exception cases, limits etc would all be duplicated/messy. Rows created in localOptimization mode are created directly in the caller's row manager, and are injected in serialized format.
+
+Why are inbound not created directly in the desired destination's allocator and then marked as serialized? Some lifespan issues... are they insurmountable?
+We do pack into dataBuffers rather than MemoryBuffers, which avoids a need to copy the data before the receiver can use it. Large rows get split and will require copying again, but we could set dataBufferSize to be bigger in localAgent mode to mitigate this somewhat.
+
+What is the lifespan issue? In-flight queries may be abandoned when a server-side query fails, times out, or no longer needs the data. Using DataBuffer does not have this issue as they are attached to the query's memory manager/allocation once read. Or we could bypass the agent queue altogether, but rather more refactoring needed for that (might almost be easier to extent the "local optimization" mode to use multiple threads at that point)
+
+abortPending, replyPending, and abortPendingData methods are unimplemented, which may lead to some inefficiencies?

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml

@@ -999,12 +999,12 @@ SEQUENTIAL(PtblOut,Bld1,Bld2,OUTPUT(FilledRecs1),OUTPUT(FilledRecs2))
     <emphasis>joincondition</emphasis> is duplicated between adjacent pairs of
     datasets, which means that this <emphasis>joincondition</emphasis>:</para>
 
-    <programlisting>       LEFT.field = RIGHT.field</programlisting>
+    <programlisting lang="ECL" role="fragment">       LEFT.field = RIGHT.field</programlisting>
 
     <para>when applied against a <emphasis>setofdatasets </emphasis>containing
     three datasets, is logically equivalent to:</para>
 
-    <programlisting>       ds1.field = ds2.field AND ds2.field = ds3.field</programlisting>
+    <programlisting lang="ECL" role="fragment">       ds1.field = ds2.field AND ds2.field = ds3.field</programlisting>
   </sect2>
 
   <sect2 id="TRANSFORM_Function_Requirements_parameters">
@@ -1101,7 +1101,7 @@ SEQUENTIAL(PtblOut,Bld1,Bld2,OUTPUT(FilledRecs1),OUTPUT(FilledRecs2))
 
     <para>Example:</para>
 
-    <programlisting id="ECL" role="runnable">Rec := RECORD,MAXLENGTH(4096)
+    <programlisting lang="ECL" role="runnable">Rec := RECORD,MAXLENGTH(4096)
   STRING1  Letter;
   UNSIGNED1    DS;
   UNSIGNED1    Matches   := 0;

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-KEYDIFF.xml

@@ -157,7 +157,7 @@
 
   <para>Example:</para>
 
-  <programlisting>Vehicles := DATASET('vehicles',
+  <programlisting lang="ECL">Vehicles := DATASET('vehicles',
      {STRING2 st,
       STRING20 city,
       STRING20 lname,

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-KEYPATCH.xml

@@ -154,7 +154,7 @@
 
   <para>Example:</para>
 
-  <programlisting>Vehicles := DATASET('vehicles',
+  <programlisting lang="ECL">Vehicles := DATASET('vehicles',
      {STRING2 st,
       STRING20 city,
       STRING20 lname,

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-KEYUNICODE.xml

@@ -45,7 +45,7 @@
 
   <para>Example:</para>
 
-  <programlisting>//where you might do this:
+  <programlisting lang="ECL">//where you might do this:
 my_record := RECORD
    UNICODE_en_US str;
 END;

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-LENGTH.xml

@@ -39,8 +39,9 @@
 
   <para>Example:</para>
 
-  <programlisting>INTEGER MyLength := LENGTH('XYZ' + 'ABC');
-      //MyLength is 6</programlisting>
+  <programlisting lang="ECL" role="runnable">INTEGER MyLength := LENGTH('XYZ' + 'ABC');
+OUTPUT(MyLength);   //MyLength is 6</programlisting>
 
-  <para>See Also: <link linkend="String_Operators">String Operators</link>, <link linkend="STRING">STRING</link></para>
+  <para>See Also: <link linkend="String_Operators">String Operators</link>,
+  <link linkend="STRING">STRING</link></para>
 </sect1>

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-LIBRARY.xml

@@ -91,7 +91,7 @@
 
   <para>Example:</para>
 
-  <programlisting>NamesRec := RECORD
+  <programlisting lang="ECL" role="notrunnable">NamesRec := RECORD
     INTEGER1  NameID;
     STRING20  FName;
     STRING20  LName;