From 1a2e915b5a32fc79fbc65ccf3de14880ab5031f3 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 1 Nov 2023 14:55:17 +0000 Subject: [PATCH] HPCC-29917 Refactor compressToBuffer to support different compression methods Also add some tests, and fix some issues that the tests showed up with small input sizes. Signed-off-by: Richard Chapman --- common/dllserver/thorplugin.cpp | 5 +- system/jlib/jfcmp.hpp | 4 - system/jlib/jlz4.cpp | 15 +-- system/jlib/jlzw.cpp | 229 +++++++++++++++++++------------- system/jlib/jlzw.hpp | 13 +- testing/unittests/jlibtests.cpp | 32 +++-- testing/unittests/unittests.cpp | 82 ++++++++++++ 7 files changed, 253 insertions(+), 127 deletions(-) diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp index 5db7bfce4e7..0fbf4079305 100644 --- a/common/dllserver/thorplugin.cpp +++ b/common/dllserver/thorplugin.cpp @@ -679,10 +679,7 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress) { mb.append((byte)0x80).append(resourceHeaderVersion); - if (compress) - compressToBuffer(mb, len, data); - else - appendToBuffer(mb, len, data); + compressToBuffer(mb, len, data, compress ? COMPRESS_METHOD_LZW : COMPRESS_METHOD_NONE); } extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data) diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp index f3d407452f3..49f4dbeb69c 100644 --- a/system/jlib/jfcmp.hpp +++ b/system/jlib/jfcmp.hpp @@ -72,8 +72,6 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf virtual void open(void *buf,size32_t max) override { - if (max<1024) - throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", max); wrmax = max; originalMax = max; if (buf) @@ -103,8 +101,6 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf { if (!initialSize) initialSize = FCMP_BUFFER_SIZE; // 1MB - if (initialSize<1024) - throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", initialSize); wrmax = initialSize; if (bufalloc) { diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp index 3b684a3bd60..59ec73bdf12 100644 --- a/system/jlib/jlz4.cpp +++ b/system/jlib/jlz4.cpp @@ -34,18 +34,17 @@ class CLZ4Compressor final : public CFcmpCompressor protected: virtual void setinmax() override { - inmax = blksz-outlen-sizeof(size32_t); - if (inmax<256) + if (blksz <= outlen+sizeof(size32_t)) trailing = true; // too small to bother compressing else { trailing = false; + inmax = blksz-outlen-sizeof(size32_t); size32_t slack = LZ4_COMPRESSBOUND(inmax) - inmax; - int inmax2 = inmax - (slack + sizeof(size32_t)); - if (inmax2<256) + if (inmax <= (slack + sizeof(size32_t))) trailing = true; else - inmax = inmax2; + inmax = inmax - (slack + sizeof(size32_t)); } } @@ -73,12 +72,6 @@ class CLZ4Compressor final : public CFcmpCompressor if (toflush == 0) return; - if (toflush < 256) - { - trailing = true; - return; - } - size32_t outSzRequired = outlen+sizeof(size32_t)*2+LZ4_COMPRESSBOUND(toflush); if (!dynamicOutSz) assertex(outSzRequired<=blksz); diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 2dbf48aca62..de32ec9a2c7 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -303,7 +303,7 @@ void CLZWCompressor::open(void *buf,size32_t max) outbuf = malloc(bufalloc); } outBufMb = NULL; - ASSERT(max>SAFETY_MARGIN+sizeof(size32_t)); // minimum required + assertex(max>SAFETY_MARGIN+sizeof(size32_t)); // minimum required maxlen=max-SAFETY_MARGIN; initCommon(); } @@ -758,82 +758,76 @@ size32_t RLEExpand(void *dst,const void *src,size32_t expsize) return (size32_t)(in-(const byte *)src); } -void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src) +void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method, const char *options) { - out.append(false); - out.append(len); - out.append(len, src); -} - -void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src) -{ - unsigned originalLength = out.length(); - out.append(true); - out.append((size32_t)0); - - if (len >= 32) + if (method != COMPRESS_METHOD_NONE && len >= 32) { + ICompressHandler *handler = queryCompressHandler(method); + if (!handler) + { + VStringBuffer s("Unknown compression method %x requested in compressToBuffer", (byte) method); + throw makeStringException(0, s.str()); + } + unsigned originalLength = out.length(); + // For back-compatibility, we always store COMPRESS_METHOD_LZW as 1 as earlier versions stored a boolean here + // rather than an enum + // This means that compressToBuffer/decompressToBuffer cannot bs used for rowdiff compression - this is not likely to be an issue + // Alternative would be a separate enum for compressToBuffer formats, but that seems more likely to cause confusion + out.append((byte) (method == COMPRESS_METHOD_LZW ? COMPRESS_METHOD_LZWLEGACY : method)); + out.append((size32_t)0); size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ... - Owned compressor = createLZWCompressor(); + Owned compressor = handler->getCompressor(options); void *newData = out.reserve(newSize); - compressor->open(newData, newSize); - if (compressor->write(src, len)==len) + try { - compressor->close(); - size32_t compressedLen = compressor->buflen(); - out.setWritePos(originalLength + sizeof(bool)); - out.append(compressedLen); - out.setWritePos(originalLength + sizeof(bool) + sizeof(size32_t) + compressedLen); - return; + compressor->open(newData, newSize); + if (compressor->write(src, len)==len) + { + compressor->close(); + size32_t compressedLen = compressor->buflen(); + out.setWritePos(originalLength + sizeof(byte)); + out.append(compressedLen); + out.setWritePos(originalLength + sizeof(byte) + sizeof(size32_t) + compressedLen); + return; + } } + catch (IException *E) + { + E->Release(); + } + // failed to compress... + out.setWritePos(originalLength); } - - // all or don't compress - out.setWritePos(originalLength); - appendToBuffer(out, len, src); -} - -void decompressToBuffer(MemoryBuffer & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.reserve(outSize); - expander->expand(buff); + out.append((byte) COMPRESS_METHOD_NONE); + out.append(len); + out.append(len, src); } - -void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in) +void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options) { - bool compressed; size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToBuffer(out, in.readDirect(srcLen)); - else + unsigned char _method; + in.read(_method).read(srcLen); + CompressionMethod method = (CompressionMethod) _method; + if (method==COMPRESS_METHOD_NONE) out.append(srcLen, in.readDirect(srcLen)); -} - -void decompressToAttr(MemoryAttr & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.allocate(outSize); - expander->expand(buff); -} - -void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in) -{ - bool compressed; - size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToAttr(out, in.readDirect(srcLen)); else - out.set(srcLen, in.readDirect(srcLen)); + { + if (method==COMPRESS_METHOD_LZWLEGACY) + method = COMPRESS_METHOD_LZW; // Back compatibilty + ICompressHandler *handler = queryCompressHandler(method); + if (!handler) + { + VStringBuffer s("Unknown decompression method %x required in decompressToBuffer", (byte) method); + throw makeStringException(0, s.str()); + } + Owned expander = handler->getExpander(options); + unsigned outSize = expander->init(in.readDirect(srcLen)); + void * buff = out.reserve(outSize); + expander->expand(buff); + } } - - /* Simple Diff compression format is @@ -1378,7 +1372,7 @@ class jlib_decl CRDiffCompressor : public ICompressor, public CInterface outbuf = malloc(bufalloc); } outBufMb = NULL; - ASSERT(max>2+sizeof(size32_t)*2); // minimum required (actually will need enough for recsize so only a guess) + assertex(max>2+sizeof(size32_t)*2); // minimum required (actually will need enough for recsize so only a guess) initCommon(); remaining = max-outlen; } @@ -1661,7 +1655,7 @@ class jlib_decl CRandRDiffCompressor : public ICompressor, public CInterface outbuf = malloc(bufalloc); } outBufMb = NULL; - ASSERT(max>MIN_RRDHEADER_SIZE+sizeof(unsigned short)+3); // hopefully a lot bigger! + assertex(max>MIN_RRDHEADER_SIZE+sizeof(unsigned short)+3); // hopefully a lot bigger! initCommon(); } @@ -2672,6 +2666,8 @@ class CAESCompressor : implements ICompressor, public CInterface else outbuf = outattr.allocate(blksize); outBufMb = NULL; + if (blksize <= AES_PADDING_SIZE+sizeof(size32_t)) + throw makeStringException(0, "CAESCompressor: target buffer too small"); size32_t subsz = blksize-AES_PADDING_SIZE-sizeof(size32_t); comp->open(compattr.reserveTruncate(subsz),subsz); } @@ -2862,44 +2858,88 @@ IPropertyTree *getBlockedFileDetails(IFile *file) return NULL; } -class CCompressHandlerArray : public IArrayOf +class CCompressHandlerArray { + IArrayOf registered; // Owns the relevant handler objects + ICompressHandler *byMethod[COMPRESS_METHOD_LAST] = { nullptr }; + ICompressHandler *AESbyMethod[COMPRESS_METHOD_LAST] = { nullptr }; + public: ICompressHandler *lookup(const char *type) const { - ForEachItemIn(h, *this) + ForEachItemIn(h, registered) { - ICompressHandler &handler = item(h); + ICompressHandler &handler = registered.item(h); if (0 == stricmp(type, handler.queryType())) return &handler; } return NULL; } + ICompressHandler *lookup(CompressionMethod method) const + { + if ((method & ~COMPRESS_METHOD_AES) >= COMPRESS_METHOD_LAST) + return nullptr; + else if (method & COMPRESS_METHOD_AES) + return AESbyMethod[method & ~COMPRESS_METHOD_AES]; + else + return byMethod[method]; + } + ICompressHandlerIterator *getIterator() + { + return new ArrayIIteratorOf, ICompressHandler, ICompressHandlerIterator>(registered); + } + bool addCompressor(ICompressHandler *handler) + { + CompressionMethod method = handler->queryMethod(); + if (lookup(method)) + { + handler->Release(); + return false; // already registered + } + registered.append(* handler); + if ((method & ~COMPRESS_METHOD_AES) < COMPRESS_METHOD_LAST) + { + if (method & COMPRESS_METHOD_AES) + AESbyMethod[method & ~COMPRESS_METHOD_AES] = handler; + else + byMethod[method] = handler; + } + return true; + } + bool removeCompressor(ICompressHandler *handler) + { + CompressionMethod method = handler->queryMethod(); + if (registered.zap(* handler)) + { + if ((method & ~COMPRESS_METHOD_AES) < COMPRESS_METHOD_LAST) + { + if (method & COMPRESS_METHOD_AES) + AESbyMethod[method & ~COMPRESS_METHOD_AES] = nullptr; + else + byMethod[method] = nullptr; + } + return true; + } + else + return false; + } } compressors; typedef IIteratorOf ICompressHandlerIterator; ICompressHandlerIterator *getCompressHandlerIterator() { - return new ArrayIIteratorOf, ICompressHandler, ICompressHandlerIterator>(compressors); + return compressors.getIterator(); } - - bool addCompressorHandler(ICompressHandler *handler) { - if (compressors.lookup(handler->queryType())) - { - handler->Release(); - return false; // already registered - } - compressors.append(* handler); - return true; + return compressors.addCompressor(handler); } bool removeCompressorHandler(ICompressHandler *handler) { - return compressors.zap(* handler); + return compressors.removeCompressor(handler); } Linked defaultCompressor; @@ -2908,38 +2948,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) { class CCompressHandlerBase : implements ICompressHandler, public CInterface { - StringAttr type; public: IMPLEMENT_IINTERFACE; - CCompressHandlerBase(const char *_type) : type(_type) { } - // ICompressHandler - virtual const char *queryType() const { return type; } }; class CFLZCompressHandler : public CCompressHandlerBase { public: - CFLZCompressHandler() : CCompressHandlerBase("FLZ") { } + virtual const char *queryType() const { return "FLZ"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_FASTLZ; } virtual ICompressor *getCompressor(const char *options) { return createFastLZCompressor(); } virtual IExpander *getExpander(const char *options) { return createFastLZExpander(); } }; class CLZ4CompressHandler : public CCompressHandlerBase { public: - CLZ4CompressHandler() : CCompressHandlerBase("LZ4") { } + virtual const char *queryType() const { return "LZ4"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, false); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CLZ4HCCompressHandler : public CCompressHandlerBase { public: - CLZ4HCCompressHandler() : CCompressHandlerBase("LZ4HC") { } + virtual const char *queryType() const { return "LZ4HC"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4HC; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, true); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CAESCompressHandler : public CCompressHandlerBase { public: - CAESCompressHandler() : CCompressHandlerBase("AES") { } + virtual const char *queryType() const { return "AES"; } + virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZW); } virtual ICompressor *getCompressor(const char *options) { assertex(options); @@ -2954,34 +2994,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) class CDiffCompressHandler : public CCompressHandlerBase { public: - CDiffCompressHandler() : CCompressHandlerBase("DIFF") { } + virtual const char *queryType() const { return "DIFF"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; } virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); } }; class CRDiffCompressHandler : public CCompressHandlerBase { public: - CRDiffCompressHandler() : CCompressHandlerBase("RDIFF") { } + virtual const char *queryType() const { return "RDIFF"; } // Synonym for DIFF + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; } virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); } }; class CRandRDiffCompressHandler : public CCompressHandlerBase { public: - CRandRDiffCompressHandler() : CCompressHandlerBase("RANDROW") { } + virtual const char *queryType() const { return "RANDROW"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_RANDROW; } virtual ICompressor *getCompressor(const char *options) { return createRandRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { UNIMPLEMENTED; } // Expander has a different interface }; class CLZWCompressHandler : public CCompressHandlerBase { public: - CLZWCompressHandler() : CCompressHandlerBase("LZW") { } + virtual const char *queryType() const { return "LZW"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZW; } virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(true); } virtual IExpander *getExpander(const char *options) { return createLZWExpander(true); } }; + addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CAESCompressHandler()); addCompressorHandler(new CDiffCompressHandler()); - addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CRDiffCompressHandler()); addCompressorHandler(new CRandRDiffCompressHandler()); addCompressorHandler(new CFLZCompressHandler()); @@ -2999,8 +3043,7 @@ ICompressHandler *queryCompressHandler(const char *type) ICompressHandler *queryCompressHandler(CompressionMethod method) { - //Could be more efficient, but doesn't matter - return compressors.lookup(translateFromCompMethod(method)); + return compressors.lookup(method); } void setDefaultCompressor(const char *type) diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp index 1d7c7196390..5581ba7abf8 100644 --- a/system/jlib/jlzw.hpp +++ b/system/jlib/jlzw.hpp @@ -35,9 +35,11 @@ enum CompressionMethod COMPRESS_METHOD_LZ4, COMPRESS_METHOD_LZ4HC, COMPRESS_METHOD_RANDROW, + COMPRESS_METHOD_LAST, COMPRESS_METHOD_AES = 0x80, + COMPRESS_METHOD_LZWLEGACY = 1, // Matches value of boolean 'true' used to indicate LZW compression by legacy compressToBuffer }; @@ -117,13 +119,9 @@ extern jlib_decl ICompressor *createRandRDiffCompressor(); // similar to RDiffCo extern jlib_decl IRandRowExpander *createRandRDiffExpander(); // NB only supports fixed row size -//Some helper functions to make it easy to compress/decompress to memorybuffers. -extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in); -extern jlib_decl void decompressToAttr(MemoryAttr & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in); -extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src); //format as failed compression +// Helper functions to make it easy to compress/decompress to memorybuffers. +extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method=COMPRESS_METHOD_LZW, const char *options=nullptr); +extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options=nullptr); interface ICompressedFileIO: extends IFileIO @@ -158,6 +156,7 @@ extern jlib_decl IPropertyTree *getBlockedFileDetails(IFile *file); interface ICompressHandler : extends IInterface { virtual const char *queryType() const = 0; + virtual CompressionMethod queryMethod() const = 0; virtual ICompressor *getCompressor(const char *options=NULL) = 0; virtual IExpander *getExpander(const char *options=NULL) = 0; }; diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 9918b77c5ef..e7178994109 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3155,7 +3155,7 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_END(); static constexpr size32_t sz = 100*0x100000; // 100MB - enum CompressOpt { RowCompress, AllRowCompress, BlockCompress }; + enum CompressOpt { RowCompress, AllRowCompress, BlockCompress, CompressToBuffer }; public: void test() { @@ -3218,6 +3218,7 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture testCompressor(handler, "hclevel=10", rowSz, src.length(), src.bytes(), RowCompress); } testCompressor(handler, options, rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, options, rowSz, src.length(), src.bytes(), CompressToBuffer); if (streq(type, "LZ4")) { testCompressor(handler, "allrow", rowSz, src.length(), src.bytes(), AllRowCompress); // block doesn't affect the compressor, just tracing @@ -3276,23 +3277,38 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture compressed.setLength(written); break; } + case CompressToBuffer: + { + compressToBuffer(compressed, srcLen, ptr, handler.queryMethod(), options); + break; + } } cycle_t compressCycles = timer.elapsedCycles(); Owned expander = handler.getExpander(options); - + MemoryBuffer tgt; timer.reset(); - size32_t required = expander->init(compressed.bytes()); - MemoryBuffer tgt(required); - expander->expand(tgt.bufferBase()); - tgt.setWritePos(required); + if (opt==CompressToBuffer) + { + decompressToBuffer(tgt, compressed, options); + } + else + { + size32_t required = expander->init(compressed.bytes()); + tgt.reserveTruncate(required); + expander->expand(tgt.bufferBase()); + tgt.setWritePos(required); + } cycle_t decompressCycles = timer.elapsedCycles(); float ratio = (float)(srcLen) / compressed.length(); StringBuffer name(handler.queryType()); - if (options) - name.append("(").append(options).append(")"); + if (opt == CompressToBuffer) + name.append("-c2b"); + if (options && *options) + name.append("-").append(options); + if (name.length() > 19) name.setLength(19); diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index a4c7e7e8ca6..6a857133192 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -1052,5 +1052,87 @@ class RelaxedAtomicTimingTest : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_REGISTRATION( RelaxedAtomicTimingTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( RelaxedAtomicTimingTest, "RelaxedAtomicTimingTest" ); +#include "jlzw.hpp" +class compressToBufferTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE( compressToBufferTest ); + CPPUNIT_TEST(testCompressors); + CPPUNIT_TEST_SUITE_END(); + + bool testOne(unsigned len, CompressionMethod method, bool prevResult, const char *options=nullptr) + { + constexpr const char *in = + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello"; + assertex(len <= strlen(in)); + MemoryBuffer compressed; + compressToBuffer(compressed, len, in, method, options); + bool ret; + if (compressed.length() == len+5) + { + if (prevResult) + DBGLOG("compressToBuffer %x size %u did not compress", (byte) method, len); + ret = false; + } + else + { + if (!prevResult) + DBGLOG("compressToBuffer %x size %u compressed to %u", (byte) method, len, compressed.length()); + ret = true; + } + CPPUNIT_ASSERT(compressed.length() <= len+5); + MemoryBuffer out; + decompressToBuffer(out, compressed, options); + CPPUNIT_ASSERT(out.length() == len); + if (len) + CPPUNIT_ASSERT(memcmp(out.bytes(), in, len) == 0); + return ret; + } + + void testCompressor(CompressionMethod method, const char *options=nullptr) + { + bool result = true; + for (unsigned i = 0; i < 256; i++) + result = testOne(i, method, result, options); + testOne(1000, method, false, options); + + } + void testCompressors() + { + testCompressor(COMPRESS_METHOD_NONE); + testCompressor(COMPRESS_METHOD_LZW); + testCompressor(COMPRESS_METHOD_LZ4); + testCompressor((CompressionMethod) (COMPRESS_METHOD_LZW|COMPRESS_METHOD_AES), "0123456789abcdef"); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( compressToBufferTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( compressToBufferTest, "CompressToBufferTest" ); + #endif // _USE_CPPUNIT