From 7854681db952ec465362a630f8b229c29dbb128d Mon Sep 17 00:00:00 2001
From: Gordon Smith <GordonJSmith@gmail.com>
Date: Thu, 29 Jun 2023 17:40:00 +0100
Subject: [PATCH 1/3] Split off 8.12.34

Signed-off-by: Gordon Smith <GordonJSmith@gmail.com>
---
 helm/hpcc/Chart.yaml                  |  4 ++--
 helm/hpcc/templates/_helpers.tpl      |  2 +-
 helm/hpcc/templates/dafilesrv.yaml    |  2 +-
 helm/hpcc/templates/dali.yaml         |  2 +-
 helm/hpcc/templates/dfuserver.yaml    |  2 +-
 helm/hpcc/templates/eclagent.yaml     |  4 ++--
 helm/hpcc/templates/eclccserver.yaml  |  4 ++--
 helm/hpcc/templates/eclscheduler.yaml |  2 +-
 helm/hpcc/templates/esp.yaml          |  2 +-
 helm/hpcc/templates/localroxie.yaml   |  2 +-
 helm/hpcc/templates/roxie.yaml        |  8 ++++----
 helm/hpcc/templates/sasha.yaml        |  2 +-
 helm/hpcc/templates/thor.yaml         | 10 +++++-----
 version.cmake                         |  2 +-
 14 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml
index fc5667f3d93..2273d116401 100644
--- a/helm/hpcc/Chart.yaml
+++ b/helm/hpcc/Chart.yaml
@@ -6,9 +6,9 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 8.12.33-closedown0
+version: 8.12.35-closedown0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 
-appVersion: 8.12.33-closedown0
+appVersion: 8.12.35-closedown0
diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
index 5e4084503f4..339d2919c32 100644
--- a/helm/hpcc/templates/_helpers.tpl
+++ b/helm/hpcc/templates/_helpers.tpl
@@ -1225,7 +1225,7 @@ kind: Service
 metadata:
   name: {{ $lvars.serviceName | quote }}
   labels:
-    helmVersion: 8.12.33-closedown0
+    helmVersion: 8.12.35-closedown0
     {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }}
 {{- if $lvars.labels }}
 {{ toYaml $lvars.labels | indent 4 }}
diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml
index d672c3fed97..08f2d729316 100644
--- a/helm/hpcc/templates/dafilesrv.yaml
+++ b/helm/hpcc/templates/dafilesrv.yaml
@@ -50,7 +50,7 @@ spec:
       labels:
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }}
         server: {{ .name | quote }}
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
       annotations:
         checksum/config: {{ $configSHA }}
     spec:
diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml
index 822df899c6f..fd7260bbb73 100644
--- a/helm/hpcc/templates/dali.yaml
+++ b/helm/hpcc/templates/dali.yaml
@@ -82,7 +82,7 @@ spec:
         run: {{ $dali.name | quote }}
         server: {{ $dali.name | quote }}
         app: dali
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey $.Values.global "metrics" }}
  {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }}
 {{- end }}
diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml
index 699ce114a3e..97218da26cf 100644
--- a/helm/hpcc/templates/dfuserver.yaml
+++ b/helm/hpcc/templates/dfuserver.yaml
@@ -56,7 +56,7 @@ spec:
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }}
         run: {{ .name | quote }}
         accessDali: "yes"
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}
diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml
index 09531fec2b8..3c30782240d 100644
--- a/helm/hpcc/templates/eclagent.yaml
+++ b/helm/hpcc/templates/eclagent.yaml
@@ -58,7 +58,7 @@ data:
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }}
             accessDali: "yes"
             accessEsp: "yes"
-            helmVersion: 8.12.33-closedown0
+            helmVersion: 8.12.35-closedown0
 {{- if hasKey .me "labels" }}
 {{ toYaml .me.labels | indent 12 }}
 {{- end }}
@@ -137,7 +137,7 @@ spec:
         run: {{ .name | quote }}
         accessDali: "yes"
         accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }}
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}
diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml
index 4d3604dd69c..cd4627f233b 100644
--- a/helm/hpcc/templates/eclccserver.yaml
+++ b/helm/hpcc/templates/eclccserver.yaml
@@ -57,7 +57,7 @@ data:
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }}
             accessDali: "yes"
             accessEsp: "yes"
-            helmVersion: 8.12.33-closedown0
+            helmVersion: 8.12.35-closedown0
 {{- if hasKey .me "labels" }}
 {{ toYaml .me.labels | indent 12 }}
 {{- end }}
@@ -142,7 +142,7 @@ spec:
         run: {{ .name | quote }}
         accessDali: "yes"
         accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }}
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}
diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml
index 527da55fd3e..0ecb65db72b 100644
--- a/helm/hpcc/templates/eclscheduler.yaml
+++ b/helm/hpcc/templates/eclscheduler.yaml
@@ -64,7 +64,7 @@ spec:
         run: {{ .name | quote }}
         accessDali: "yes"
         accessEsp: "no"
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}
diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml
index f540eda2fe7..9c84194a291 100644
--- a/helm/hpcc/templates/esp.yaml
+++ b/helm/hpcc/templates/esp.yaml
@@ -117,7 +117,7 @@ spec:
         server: {{ .name | quote }}
         accessDali: "yes"
         app: {{ $application }}
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
         {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }}
 {{- if hasKey $.Values.global "metrics" }}
  {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }}
diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml
index 37cdd0fd634..acbe0e0fc05 100644
--- a/helm/hpcc/templates/localroxie.yaml
+++ b/helm/hpcc/templates/localroxie.yaml
@@ -70,7 +70,7 @@ spec:
         server: {{ $servername | quote }}
         accessDali: "yes"
         accessEsp: "yes"
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }}
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml
index d26f219a46c..f0384c340f6 100644
--- a/helm/hpcc/templates/roxie.yaml
+++ b/helm/hpcc/templates/roxie.yaml
@@ -120,7 +120,7 @@ spec:
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }}
         run: {{ $commonCtx.toponame | quote }}
         roxie-cluster: {{ $roxie.name | quote }}
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey $.Values.global "metrics" }}
  {{- include "hpcc.generateMetricsReporterLabel"  $.Values.global.metrics | nindent 8}}
 {{- end }}
@@ -180,7 +180,7 @@ kind: Service
 metadata:
   name: {{ $commonCtx.toponame | quote }}
   labels:
-    helmVersion: 8.12.33-closedown0
+    helmVersion: 8.12.35-closedown0
     {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }}
 spec:
   ports:
@@ -242,7 +242,7 @@ spec:
         roxie-cluster: {{ $roxie.name | quote }}
         accessDali: "yes"
         accessEsp: "yes"
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }}
 {{- if hasKey $.Values.global "metrics" }}
  {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}}
@@ -345,7 +345,7 @@ spec:
         roxie-cluster: {{ $roxie.name | quote }}
         accessDali: "yes"
         accessEsp: "yes"
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey $.Values.global "metrics" }}
  {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}}
 {{- end }}
diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml
index 2c0cb79550e..3ff027050fe 100644
--- a/helm/hpcc/templates/sasha.yaml
+++ b/helm/hpcc/templates/sasha.yaml
@@ -52,7 +52,7 @@ spec:
         run: {{ $serviceName | quote }}
         server: {{ $serviceName | quote }}
         accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }}
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
 {{- if hasKey $sasha "labels" }}
 {{ toYaml $sasha.labels | indent 8 }}
 {{- end }}
diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml
index a2636be6ec0..d7d896d8666 100644
--- a/helm/hpcc/templates/thor.yaml
+++ b/helm/hpcc/templates/thor.yaml
@@ -82,7 +82,7 @@ data:
           labels:
             accessDali: "yes"
             accessEsp: "yes"
-            helmVersion: 8.12.33-closedown0
+            helmVersion: 8.12.35-closedown0
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }}
 {{- if hasKey .me "labels" }}
 {{ toYaml .me.labels | indent 12 }}
@@ -149,7 +149,7 @@ data:
             accessEsp: "yes"
             app: "thor"
             component: "thormanager"
-            helmVersion: 8.12.33-closedown0
+            helmVersion: 8.12.35-closedown0
             instance: "_HPCC_JOBNAME_"
             job: "_HPCC_JOBNAME_"
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }}
@@ -218,7 +218,7 @@ data:
             accessEsp: "yes"
             app: "thor"
             component: "thorworker"
-            helmVersion: 8.12.33-closedown0
+            helmVersion: 8.12.35-closedown0
             instance: "_HPCC_JOBNAME_"
             job: "_HPCC_JOBNAME_"
             {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }}
@@ -353,7 +353,7 @@ spec:
         accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }}
         app: "thor"
         component: "thor-eclagent"
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
         instance: {{ $commonCtx.eclAgentName | quote }}
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }}
 {{- if hasKey $commonCtx.me "labels" }}
@@ -418,7 +418,7 @@ spec:
         accessEsp: "no"
         app: "thor"
         component: "thor-thoragent"
-        helmVersion: 8.12.33-closedown0
+        helmVersion: 8.12.35-closedown0
         instance: {{ $commonCtx.thorAgentName | quote }}
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }}
 {{- if hasKey $commonCtx.me "labels" }}
diff --git a/version.cmake b/version.cmake
index ec2b1f03cba..430984aeb8d 100644
--- a/version.cmake
+++ b/version.cmake
@@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" )
 set ( HPCC_PROJECT "community" )
 set ( HPCC_MAJOR 8 )
 set ( HPCC_MINOR 12 )
-set ( HPCC_POINT 33 )
+set ( HPCC_POINT 35 )
 set ( HPCC_MATURITY "closedown" )
 set ( HPCC_SEQUENCE 0 )
 ###

From 0e715d8f73e18c9f0f27030e283e5f8c5288f534 Mon Sep 17 00:00:00 2001
From: Jake Smith <jake.smith@lexisnexisrisk.com>
Date: Wed, 15 Mar 2023 18:54:00 +0000
Subject: [PATCH 2/3] HPCC-29187 File updater shutdown crash fix

A regression introduced by HPCC-28968 caused removeConfigUpdateHook
to crash accessing released configFileUpdater.
Fix by ensuring configFileUpdater set to null after releasing.

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>
---
 system/jlib/jptree.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp
index aacba5fd87c..45f72781978 100644
--- a/system/jlib/jptree.cpp
+++ b/system/jlib/jptree.cpp
@@ -8802,6 +8802,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
 MODULE_EXIT()
 {
     ::Release(configFileUpdater);
+    configFileUpdater = nullptr;
 }
 
 unsigned installConfigUpdateHook(ConfigUpdateFunc notifyFunc)

From 97d19f4058163671dac984782e9f52c0708e0027 Mon Sep 17 00:00:00 2001
From: Gavin Halliday <gavin.halliday@lexisnexis.com>
Date: Wed, 5 Jul 2023 18:15:42 +0100
Subject: [PATCH 3/3] HPCC-29385 Allow incremental decompression from lz4 files

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>
---
 system/jlib/jfcmp.hpp | 46 +------------------------
 system/jlib/jflz.cpp  |  2 +-
 system/jlib/jlz4.cpp  | 63 +++++++++++++++++++++++++++++++++-
 system/jlib/jlzw.cpp  | 79 ++++++++++++++++++++++++++++++++++++-------
 system/jlib/jlzw.hpp  |  9 +++++
 system/jlib/jlzw.ipp  |  6 +---
 6 files changed, 141 insertions(+), 64 deletions(-)

diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp
index 3e5b96ce618..e1fc4aad577 100644
--- a/system/jlib/jfcmp.hpp
+++ b/system/jlib/jfcmp.hpp
@@ -211,7 +211,7 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf<ICompressor>
 };
 
 
-class jlib_decl CFcmpExpander : public CSimpleInterfaceOf<IExpander>
+class jlib_decl CFcmpExpander : public CExpanderBase
 {
 protected:
     byte *outbuf;
@@ -241,50 +241,6 @@ class jlib_decl CFcmpExpander : public CSimpleInterfaceOf<IExpander>
         return outlen;
     }
 
-    virtual void expand(void *buf)
-    {
-        if (!outlen)
-            return;
-        if (buf)
-        {
-            if (bufalloc)
-                free(outbuf);
-            bufalloc = 0;
-            outbuf = (unsigned char *)buf;
-        }
-        else if (outlen>bufalloc)
-        {
-            if (bufalloc)
-                free(outbuf);
-            bufalloc = outlen;
-            outbuf = (unsigned char *)malloc(bufalloc);
-            if (!outbuf)
-                throw MakeStringException(MSGAUD_operator,0, "Out of memory in FcmpExpander::expand, requesting %d bytes", bufalloc);
-        }
-        size32_t done = 0;
-        for (;;)
-        {
-            const size32_t szchunk = *in;
-            in++;
-            if (szchunk+done<outlen)
-            {
-                memcpy((byte *)buf+done, in, szchunk);
-                size32_t written = szchunk;
-                done += written;
-                if (!written||(done>outlen))
-                    throw MakeStringException(0, "FcmpExpander - corrupt data(1) %d %d",written,szchunk);
-            }
-            else
-            {
-                if (szchunk+done!=outlen)
-                    throw MakeStringException(0, "FcmpExpander - corrupt data(2) %d %d",szchunk,outlen);
-                memcpy((byte *)buf+done,in,szchunk);
-                break;
-            }
-            in = (const size32_t *)(((const byte *)in)+szchunk);
-        }
-    }
-
     virtual void *bufptr() { return outbuf;}
     virtual size32_t buflen() { return outlen;}
 };
diff --git a/system/jlib/jflz.cpp b/system/jlib/jflz.cpp
index a809964c540..078229722ba 100644
--- a/system/jlib/jflz.cpp
+++ b/system/jlib/jflz.cpp
@@ -687,7 +687,7 @@ class CFastLZCompressor final : public CFcmpCompressor
 class jlib_decl CFastLZExpander : public CFcmpExpander
 {
 public:
-    virtual void expand(void *buf)
+    virtual void expand(void *buf) override
     {
         if (!outlen)
             return;
diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp
index 92f3a8134d6..3b684a3bd60 100644
--- a/system/jlib/jlz4.cpp
+++ b/system/jlib/jlz4.cpp
@@ -177,8 +177,9 @@ class CLZ4Compressor final : public CFcmpCompressor
 
 class jlib_decl CLZ4Expander : public CFcmpExpander
 {
+    size32_t totalExpanded = 0;
 public:
-    virtual void expand(void *buf)
+    virtual void expand(void *buf) override
     {
         if (!outlen)
             return;
@@ -221,6 +222,66 @@ class jlib_decl CLZ4Expander : public CFcmpExpander
         }
     }
 
+    virtual size32_t expandFirst(MemoryBuffer & target, const void * src) override
+    {
+        init(src);
+        totalExpanded = 0;
+        return expandNext(target);
+    }
+
+    virtual size32_t expandNext(MemoryBuffer & target) override
+    {
+        if (totalExpanded == outlen)
+            return 0;
+
+        const size32_t szchunk = *in;
+        in++;
+
+        target.clear();
+        size32_t written;
+        if (szchunk+totalExpanded<outlen)
+        {
+            //All but the last block are compressed (see expand() function above).
+            //Slightly concerning there always has to be one trailing byte for this to work!
+            size32_t maxOut = target.capacity();
+            size32_t maxEstimate = (outlen - totalExpanded);
+            size32_t estimate = szchunk; // start conservatively - likely to be preallocated to correct size already.
+            if (estimate > maxEstimate)
+                estimate = maxEstimate;
+            if (maxOut < estimate)
+                maxOut = estimate;
+
+            for (;;)
+            {
+                //Try and compress into the current target buffer.  If too small increase size and repeat
+                written = LZ4_decompress_safe((const char *)in, (char *)target.reserve(maxOut), szchunk, maxOut);
+                if ((int)written > 0)
+                {
+                    target.setLength(written);
+                    break;
+                }
+
+                //Sanity check to catch corrupt lz4 data that always returns an error.
+                if (maxOut > outlen)
+                    throwUnexpected();
+
+                maxOut += szchunk; // Likely to quickly approach the actual expanded size
+                target.clear();
+            }
+        }
+        else
+        {
+            void * buf = target.reserve(szchunk);
+            written = szchunk;
+            memcpy(buf,in,szchunk);
+        }
+
+        in = (const size32_t *)(((const byte *)in)+szchunk);
+        totalExpanded += written;
+        if (totalExpanded > outlen)
+            throw MakeStringException(0, "LZ4Expander - corrupt data(3) %d %d",written,szchunk);
+        return written;
+    }
 };
 
 void LZ4CompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp
index 8e343cecb6d..46604389dcf 100644
--- a/system/jlib/jlzw.cpp
+++ b/system/jlib/jlzw.cpp
@@ -448,6 +448,20 @@ void CLZWCompressor::close()
     }
 }
 
+
+size32_t CExpanderBase::expandFirst(MemoryBuffer & target, const void * src)
+{
+    size32_t size = init(src);
+    void * buffer = target.reserve(size);
+    expand(buffer);
+    return size;
+}
+
+size32_t CExpanderBase::expandNext(MemoryBuffer & target)
+{
+    return 0;
+}
+
 CLZWExpander::CLZWExpander(bool _supportbigendian)
 {
     outbuf = NULL;
@@ -1467,7 +1481,7 @@ class jlib_decl CRDiffCompressor : public ICompressor, public CInterface
 };
 
 
-class jlib_decl CRDiffExpander : public IExpander, public CInterface
+class jlib_decl CRDiffExpander : public CExpanderBase
 {
     unsigned char *outbuf;
     size32_t outlen;
@@ -1475,8 +1489,6 @@ class jlib_decl CRDiffExpander : public IExpander, public CInterface
     unsigned char *in;
     size32_t recsize;
 public:
-    IMPLEMENT_IINTERFACE;
-
     CRDiffExpander()
     {
         outbuf = NULL;
@@ -1987,8 +1999,12 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
     bool writeException;
     Owned<ICompressor> compressor;
     Owned<IExpander> expander;
+    MemoryAttr compressedInputBlock;
     unsigned compMethod;
     offset_t lastFlushPos = (offset_t)-1;
+    offset_t nextExpansionPos = (offset_t)-1;
+    offset_t startBlockPos = (offset_t)-1;
+    size32_t fullBlockSize = 0;
 
     unsigned indexNum() { return indexbuf.length()/sizeof(offset_t); }
 
@@ -2017,6 +2033,43 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
     void getblock(offset_t pos)
     {
         curblockbuf.clear();
+
+        //If the blocks are being expanded incrementally check if the position is within the current block
+        //This test will never be true for row compressed data, or non-incremental decompression
+        if ((pos >= startBlockPos) && (pos < startBlockPos + fullBlockSize))
+        {
+            if (pos < nextExpansionPos)
+            {
+                //Start decompressing again and avoid re-reading the data from disk
+                const void * rawData;
+                if (fileio)
+                    rawData = compressedInputBlock.get();
+                else
+                    rawData = mmfile->base()+startBlockPos;
+
+                assertex(rawData);
+                size32_t exp = expander->expandFirst(curblockbuf, rawData);
+                curblockpos = startBlockPos;
+                nextExpansionPos = startBlockPos + exp;
+                if (pos < nextExpansionPos)
+                    return;
+
+                curblockbuf.clear();
+            }
+
+            for (;;)
+            {
+                size32_t nextSize = expander->expandNext(curblockbuf);
+                if (nextSize == 0)
+                    throwUnexpected(); // Should have failed the outer block test if nextSize is 0
+
+                curblockpos = nextExpansionPos;
+                nextExpansionPos = nextExpansionPos+nextSize;
+                if (pos < nextExpansionPos)
+                    return;
+            }
+        }
+
         size32_t expsize;
         curblocknum = lookupIndex(pos,curblockpos,expsize);
         size32_t toread = trailer.blockSize;
@@ -2027,8 +2080,9 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
         if (!toread) 
             return;
         if (fileio) {
-            MemoryAttr comp;
-            void *b=comp.allocate(toread);
+            //Allocate on the first call, reuse on subsequent calls.
+            void * b = compressedInputBlock.allocate(trailer.blockSize);
+
             size32_t r = fileio->read(p,toread,b);
             assertex(r==toread);
             expand(b,curblockbuf,expsize);
@@ -2070,11 +2124,10 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
         }
         else { // lzw or fastlz or lz4
             assertex(expander.get());
-            size32_t exp = expander->init(compbuf);
-            if (exp!=expsize) {
-                throw MakeStringException(-1,"Compressed file format failure(%d,%d) - Encrypted?",exp,expsize);
-            }
-            expander->expand(expbuf.reserve(exp));
+            size32_t exp = expander->expandFirst(expbuf, compbuf);
+            startBlockPos = curblockpos;
+            nextExpansionPos = startBlockPos + exp;
+            fullBlockSize = expsize;
         }
     }
 
@@ -2224,6 +2277,9 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
                         compMethod = COMPRESS_METHOD_LZW;
                         expander.setown(createLZWExpander(true));
                     }
+                    //Preallocate the expansion target to the block size - to ensure it is the right size and
+                    //avoid reallocation when expanding lz4
+                    curblockbuf.ensureCapacity(trailer.blockSize);
                 }
             }
         }
@@ -2685,13 +2741,12 @@ class CAESCompressor : implements ICompressor, public CInterface
     virtual CompressionMethod getCompressionMethod() const override { return (CompressionMethod)(COMPRESS_METHOD_AES | comp->getCompressionMethod()); }
 };
 
-class CAESExpander : implements IExpander, public CInterface
+class CAESExpander : implements CExpanderBase
 {
     Owned<IExpander> exp;   // base expander
     MemoryBuffer compbuf;
     MemoryAttr key;
 public:
-    IMPLEMENT_IINTERFACE;
     CAESExpander(const void *_key, unsigned _keylen)
         : key(_keylen,_key)
     {
diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp
index 72816bc4370..1d7c7196390 100644
--- a/system/jlib/jlzw.hpp
+++ b/system/jlib/jlzw.hpp
@@ -66,6 +66,8 @@ interface jlib_decl IExpander : public IInterface
     virtual void   expand(void *target)=0;
     virtual void * bufptr()=0;
     virtual size32_t buflen()=0;
+    virtual size32_t expandFirst(MemoryBuffer & target, const void * src) = 0;
+    virtual size32_t expandNext(MemoryBuffer & target) = 0;
 };
 
 
@@ -82,6 +84,13 @@ interface jlib_decl IRandRowExpander : public IInterface
 };
 
 
+class jlib_decl CExpanderBase : public CInterfaceOf<IExpander>
+{
+public:
+    //Provide default implementations
+    virtual size32_t expandFirst(MemoryBuffer & target, const void * src) override;
+    virtual size32_t expandNext(MemoryBuffer & target) override;
+};
 
 
 extern jlib_decl ICompressor *createLZWCompressor(bool supportbigendian=false); // bigendiansupport required for cross platform with solaris
diff --git a/system/jlib/jlzw.ipp b/system/jlib/jlzw.ipp
index 0e4ce4fcf7a..1f562b7d6cf 100644
--- a/system/jlib/jlzw.ipp
+++ b/system/jlib/jlzw.ipp
@@ -38,7 +38,6 @@ public:
     unsigned char dictchar[LZW_HASH_TABLE_SIZE];
 };
 
-
 class CLZWCompressor final : public ICompressor, public CInterface
 {
 public:
@@ -88,12 +87,9 @@ protected:
     bool supportbigendian;
 };
 
-
-class jlib_decl CLZWExpander : public IExpander, public CInterface
+class  CLZWExpander : public CExpanderBase
 {
 public:
-    IMPLEMENT_IINTERFACE;
-
     CLZWExpander(bool _supportbigendian);
     ~CLZWExpander();
     virtual size32_t  init(const void *blk); // returns size required