From 7854681db952ec465362a630f8b229c29dbb128d Mon Sep 17 00:00:00 2001 From: Gordon Smith <GordonJSmith@gmail.com> Date: Thu, 29 Jun 2023 17:40:00 +0100 Subject: [PATCH 1/3] Split off 8.12.34 Signed-off-by: Gordon Smith <GordonJSmith@gmail.com> --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index fc5667f3d93..2273d116401 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.33-closedown0 +version: 8.12.35-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.33-closedown0 +appVersion: 8.12.35-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 5e4084503f4..339d2919c32 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1225,7 +1225,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index d672c3fed97..08f2d729316 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 822df899c6f..fd7260bbb73 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 699ce114a3e..97218da26cf 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 09531fec2b8..3c30782240d 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 4d3604dd69c..cd4627f233b 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 527da55fd3e..0ecb65db72b 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index f540eda2fe7..9c84194a291 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 37cdd0fd634..acbe0e0fc05 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index d26f219a46c..f0384c340f6 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -345,7 +345,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 2c0cb79550e..3ff027050fe 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index a2636be6ec0..d7d896d8666 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.33-closedown0 + helmVersion: 8.12.35-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index ec2b1f03cba..430984aeb8d 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 33 ) +set ( HPCC_POINT 35 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 0e715d8f73e18c9f0f27030e283e5f8c5288f534 Mon Sep 17 00:00:00 2001 From: Jake Smith <jake.smith@lexisnexisrisk.com> Date: Wed, 15 Mar 2023 18:54:00 +0000 Subject: [PATCH 2/3] HPCC-29187 File updater shutdown crash fix A regression introduced by HPCC-28968 caused removeConfigUpdateHook to crash accessing released configFileUpdater. Fix by ensuring configFileUpdater set to null after releasing. Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com> --- system/jlib/jptree.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index aacba5fd87c..45f72781978 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -8802,6 +8802,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) MODULE_EXIT() { ::Release(configFileUpdater); + configFileUpdater = nullptr; } unsigned installConfigUpdateHook(ConfigUpdateFunc notifyFunc) From 97d19f4058163671dac984782e9f52c0708e0027 Mon Sep 17 00:00:00 2001 From: Gavin Halliday <gavin.halliday@lexisnexis.com> Date: Wed, 5 Jul 2023 18:15:42 +0100 Subject: [PATCH 3/3] HPCC-29385 Allow incremental decompression from lz4 files Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com> --- system/jlib/jfcmp.hpp | 46 +------------------------ system/jlib/jflz.cpp | 2 +- system/jlib/jlz4.cpp | 63 +++++++++++++++++++++++++++++++++- system/jlib/jlzw.cpp | 79 ++++++++++++++++++++++++++++++++++++------- system/jlib/jlzw.hpp | 9 +++++ system/jlib/jlzw.ipp | 6 +--- 6 files changed, 141 insertions(+), 64 deletions(-) diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp index 3e5b96ce618..e1fc4aad577 100644 --- a/system/jlib/jfcmp.hpp +++ b/system/jlib/jfcmp.hpp @@ -211,7 +211,7 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf<ICompressor> }; -class jlib_decl CFcmpExpander : public CSimpleInterfaceOf<IExpander> +class jlib_decl CFcmpExpander : public CExpanderBase { protected: byte *outbuf; @@ -241,50 +241,6 @@ class jlib_decl CFcmpExpander : public CSimpleInterfaceOf<IExpander> return outlen; } - virtual void expand(void *buf) - { - if (!outlen) - return; - if (buf) - { - if (bufalloc) - free(outbuf); - bufalloc = 0; - outbuf = (unsigned char *)buf; - } - else if (outlen>bufalloc) - { - if (bufalloc) - free(outbuf); - bufalloc = outlen; - outbuf = (unsigned char *)malloc(bufalloc); - if (!outbuf) - throw MakeStringException(MSGAUD_operator,0, "Out of memory in FcmpExpander::expand, requesting %d bytes", bufalloc); - } - size32_t done = 0; - for (;;) - { - const size32_t szchunk = *in; - in++; - if (szchunk+done<outlen) - { - memcpy((byte *)buf+done, in, szchunk); - size32_t written = szchunk; - done += written; - if (!written||(done>outlen)) - throw MakeStringException(0, "FcmpExpander - corrupt data(1) %d %d",written,szchunk); - } - else - { - if (szchunk+done!=outlen) - throw MakeStringException(0, "FcmpExpander - corrupt data(2) %d %d",szchunk,outlen); - memcpy((byte *)buf+done,in,szchunk); - break; - } - in = (const size32_t *)(((const byte *)in)+szchunk); - } - } - virtual void *bufptr() { return outbuf;} virtual size32_t buflen() { return outlen;} }; diff --git a/system/jlib/jflz.cpp b/system/jlib/jflz.cpp index a809964c540..078229722ba 100644 --- a/system/jlib/jflz.cpp +++ b/system/jlib/jflz.cpp @@ -687,7 +687,7 @@ class CFastLZCompressor final : public CFcmpCompressor class jlib_decl CFastLZExpander : public CFcmpExpander { public: - virtual void expand(void *buf) + virtual void expand(void *buf) override { if (!outlen) return; diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp index 92f3a8134d6..3b684a3bd60 100644 --- a/system/jlib/jlz4.cpp +++ b/system/jlib/jlz4.cpp @@ -177,8 +177,9 @@ class CLZ4Compressor final : public CFcmpCompressor class jlib_decl CLZ4Expander : public CFcmpExpander { + size32_t totalExpanded = 0; public: - virtual void expand(void *buf) + virtual void expand(void *buf) override { if (!outlen) return; @@ -221,6 +222,66 @@ class jlib_decl CLZ4Expander : public CFcmpExpander } } + virtual size32_t expandFirst(MemoryBuffer & target, const void * src) override + { + init(src); + totalExpanded = 0; + return expandNext(target); + } + + virtual size32_t expandNext(MemoryBuffer & target) override + { + if (totalExpanded == outlen) + return 0; + + const size32_t szchunk = *in; + in++; + + target.clear(); + size32_t written; + if (szchunk+totalExpanded<outlen) + { + //All but the last block are compressed (see expand() function above). + //Slightly concerning there always has to be one trailing byte for this to work! + size32_t maxOut = target.capacity(); + size32_t maxEstimate = (outlen - totalExpanded); + size32_t estimate = szchunk; // start conservatively - likely to be preallocated to correct size already. + if (estimate > maxEstimate) + estimate = maxEstimate; + if (maxOut < estimate) + maxOut = estimate; + + for (;;) + { + //Try and compress into the current target buffer. If too small increase size and repeat + written = LZ4_decompress_safe((const char *)in, (char *)target.reserve(maxOut), szchunk, maxOut); + if ((int)written > 0) + { + target.setLength(written); + break; + } + + //Sanity check to catch corrupt lz4 data that always returns an error. + if (maxOut > outlen) + throwUnexpected(); + + maxOut += szchunk; // Likely to quickly approach the actual expanded size + target.clear(); + } + } + else + { + void * buf = target.reserve(szchunk); + written = szchunk; + memcpy(buf,in,szchunk); + } + + in = (const size32_t *)(((const byte *)in)+szchunk); + totalExpanded += written; + if (totalExpanded > outlen) + throw MakeStringException(0, "LZ4Expander - corrupt data(3) %d %d",written,szchunk); + return written; + } }; void LZ4CompressToBuffer(MemoryBuffer & out, size32_t len, const void * src) diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 8e343cecb6d..46604389dcf 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -448,6 +448,20 @@ void CLZWCompressor::close() } } + +size32_t CExpanderBase::expandFirst(MemoryBuffer & target, const void * src) +{ + size32_t size = init(src); + void * buffer = target.reserve(size); + expand(buffer); + return size; +} + +size32_t CExpanderBase::expandNext(MemoryBuffer & target) +{ + return 0; +} + CLZWExpander::CLZWExpander(bool _supportbigendian) { outbuf = NULL; @@ -1467,7 +1481,7 @@ class jlib_decl CRDiffCompressor : public ICompressor, public CInterface }; -class jlib_decl CRDiffExpander : public IExpander, public CInterface +class jlib_decl CRDiffExpander : public CExpanderBase { unsigned char *outbuf; size32_t outlen; @@ -1475,8 +1489,6 @@ class jlib_decl CRDiffExpander : public IExpander, public CInterface unsigned char *in; size32_t recsize; public: - IMPLEMENT_IINTERFACE; - CRDiffExpander() { outbuf = NULL; @@ -1987,8 +1999,12 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface bool writeException; Owned<ICompressor> compressor; Owned<IExpander> expander; + MemoryAttr compressedInputBlock; unsigned compMethod; offset_t lastFlushPos = (offset_t)-1; + offset_t nextExpansionPos = (offset_t)-1; + offset_t startBlockPos = (offset_t)-1; + size32_t fullBlockSize = 0; unsigned indexNum() { return indexbuf.length()/sizeof(offset_t); } @@ -2017,6 +2033,43 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface void getblock(offset_t pos) { curblockbuf.clear(); + + //If the blocks are being expanded incrementally check if the position is within the current block + //This test will never be true for row compressed data, or non-incremental decompression + if ((pos >= startBlockPos) && (pos < startBlockPos + fullBlockSize)) + { + if (pos < nextExpansionPos) + { + //Start decompressing again and avoid re-reading the data from disk + const void * rawData; + if (fileio) + rawData = compressedInputBlock.get(); + else + rawData = mmfile->base()+startBlockPos; + + assertex(rawData); + size32_t exp = expander->expandFirst(curblockbuf, rawData); + curblockpos = startBlockPos; + nextExpansionPos = startBlockPos + exp; + if (pos < nextExpansionPos) + return; + + curblockbuf.clear(); + } + + for (;;) + { + size32_t nextSize = expander->expandNext(curblockbuf); + if (nextSize == 0) + throwUnexpected(); // Should have failed the outer block test if nextSize is 0 + + curblockpos = nextExpansionPos; + nextExpansionPos = nextExpansionPos+nextSize; + if (pos < nextExpansionPos) + return; + } + } + size32_t expsize; curblocknum = lookupIndex(pos,curblockpos,expsize); size32_t toread = trailer.blockSize; @@ -2027,8 +2080,9 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface if (!toread) return; if (fileio) { - MemoryAttr comp; - void *b=comp.allocate(toread); + //Allocate on the first call, reuse on subsequent calls. + void * b = compressedInputBlock.allocate(trailer.blockSize); + size32_t r = fileio->read(p,toread,b); assertex(r==toread); expand(b,curblockbuf,expsize); @@ -2070,11 +2124,10 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface } else { // lzw or fastlz or lz4 assertex(expander.get()); - size32_t exp = expander->init(compbuf); - if (exp!=expsize) { - throw MakeStringException(-1,"Compressed file format failure(%d,%d) - Encrypted?",exp,expsize); - } - expander->expand(expbuf.reserve(exp)); + size32_t exp = expander->expandFirst(expbuf, compbuf); + startBlockPos = curblockpos; + nextExpansionPos = startBlockPos + exp; + fullBlockSize = expsize; } } @@ -2224,6 +2277,9 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface compMethod = COMPRESS_METHOD_LZW; expander.setown(createLZWExpander(true)); } + //Preallocate the expansion target to the block size - to ensure it is the right size and + //avoid reallocation when expanding lz4 + curblockbuf.ensureCapacity(trailer.blockSize); } } } @@ -2685,13 +2741,12 @@ class CAESCompressor : implements ICompressor, public CInterface virtual CompressionMethod getCompressionMethod() const override { return (CompressionMethod)(COMPRESS_METHOD_AES | comp->getCompressionMethod()); } }; -class CAESExpander : implements IExpander, public CInterface +class CAESExpander : implements CExpanderBase { Owned<IExpander> exp; // base expander MemoryBuffer compbuf; MemoryAttr key; public: - IMPLEMENT_IINTERFACE; CAESExpander(const void *_key, unsigned _keylen) : key(_keylen,_key) { diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp index 72816bc4370..1d7c7196390 100644 --- a/system/jlib/jlzw.hpp +++ b/system/jlib/jlzw.hpp @@ -66,6 +66,8 @@ interface jlib_decl IExpander : public IInterface virtual void expand(void *target)=0; virtual void * bufptr()=0; virtual size32_t buflen()=0; + virtual size32_t expandFirst(MemoryBuffer & target, const void * src) = 0; + virtual size32_t expandNext(MemoryBuffer & target) = 0; }; @@ -82,6 +84,13 @@ interface jlib_decl IRandRowExpander : public IInterface }; +class jlib_decl CExpanderBase : public CInterfaceOf<IExpander> +{ +public: + //Provide default implementations + virtual size32_t expandFirst(MemoryBuffer & target, const void * src) override; + virtual size32_t expandNext(MemoryBuffer & target) override; +}; extern jlib_decl ICompressor *createLZWCompressor(bool supportbigendian=false); // bigendiansupport required for cross platform with solaris diff --git a/system/jlib/jlzw.ipp b/system/jlib/jlzw.ipp index 0e4ce4fcf7a..1f562b7d6cf 100644 --- a/system/jlib/jlzw.ipp +++ b/system/jlib/jlzw.ipp @@ -38,7 +38,6 @@ public: unsigned char dictchar[LZW_HASH_TABLE_SIZE]; }; - class CLZWCompressor final : public ICompressor, public CInterface { public: @@ -88,12 +87,9 @@ protected: bool supportbigendian; }; - -class jlib_decl CLZWExpander : public IExpander, public CInterface +class CLZWExpander : public CExpanderBase { public: - IMPLEMENT_IINTERFACE; - CLZWExpander(bool _supportbigendian); ~CLZWExpander(); virtual size32_t init(const void *blk); // returns size required