diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp index d43792b4743..3fcfc797c03 100644 --- a/common/dllserver/thorplugin.cpp +++ b/common/dllserver/thorplugin.cpp @@ -676,10 +676,7 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress) { mb.append((byte)0x80).append(resourceHeaderVersion); - if (compress) - compressToBuffer(mb, len, data); - else - appendToBuffer(mb, len, data); + compressToBuffer(mb, len, data, compress ? COMPRESS_METHOD_LZW : COMPRESS_METHOD_NONE); } extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data) diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp index 0543275cf97..40768cd9d9c 100644 --- a/system/jlib/jfcmp.hpp +++ b/system/jlib/jfcmp.hpp @@ -72,8 +72,8 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf virtual void open(void *buf,size32_t max) { - if (max<1024) - throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", max); +// if (max<1024) +// throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", max); wrmax = max; originalMax = max; if (buf) @@ -103,8 +103,8 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf { if (!initialSize) initialSize = FCMP_BUFFER_SIZE; // 1MB - if (initialSize<1024) - throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", initialSize); + //if (initialSize<1024) + // throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", initialSize); wrmax = initialSize; if (bufalloc) { diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp index 1cfb86f17f7..41816cfb555 100644 --- a/system/jlib/jlz4.cpp +++ b/system/jlib/jlz4.cpp @@ -33,18 +33,17 @@ class CLZ4Compressor final : public CFcmpCompressor protected: virtual void setinmax() override { - inmax = blksz-outlen-sizeof(size32_t); - if (inmax<256) + if (blksz <= outlen+sizeof(size32_t)) trailing = true; // too small to bother compressing else { trailing = false; + inmax = blksz-outlen-sizeof(size32_t); size32_t slack = LZ4_COMPRESSBOUND(inmax) - inmax; - int inmax2 = inmax - (slack + sizeof(size32_t)); - if (inmax2<256) + if (inmax <= (slack + sizeof(size32_t))) trailing = true; else - inmax = inmax2; + inmax = inmax - (slack + sizeof(size32_t)); } } @@ -72,7 +71,7 @@ class CLZ4Compressor final : public CFcmpCompressor if (toflush == 0) return; - if (toflush < 256) + if (false && toflush < 256) { trailing = true; return; diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 2bfa1f29e9f..ffa5b8f1cd5 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -744,82 +744,76 @@ size32_t RLEExpand(void *dst,const void *src,size32_t expsize) return (size32_t)(in-(const byte *)src); } -void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src) +void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method, const char *options) { - out.append(false); - out.append(len); - out.append(len, src); -} - -void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src) -{ - unsigned originalLength = out.length(); - out.append(true); - out.append((size32_t)0); - - if (len >= 32) + if (method != COMPRESS_METHOD_NONE && len >= 32) { + ICompressHandler *handler = queryCompressHandler(method); + if (!handler) + { + VStringBuffer s("Unknown compression method %x requested in compressToBuffer", (byte) method); + throw makeStringException(0, s.str()); + } + unsigned originalLength = out.length(); + // For back-compatibility, we always store COMPRESS_METHOD_LZW as 1 as earlier versions stored a boolean here + // rather than an enum + // This means that compressToBuffer/decompressToBuffer cannot bs used for rowdiff compression - this is not likely to be an issue + // Alternative would be a separate enum for compressToBuffer formats, but that seems more likely to cause confusion + out.append((byte) (method == COMPRESS_METHOD_LZW ? COMPRESS_METHOD_LZWLEGACY : method)); + out.append((size32_t)0); size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ... - Owned compressor = createLZWCompressor(); + Owned compressor = handler->getCompressor(options); void *newData = out.reserve(newSize); - compressor->open(newData, newSize); - if (compressor->write(src, len)==len) + try { - compressor->close(); - size32_t compressedLen = compressor->buflen(); - out.setWritePos(originalLength + sizeof(bool)); - out.append(compressedLen); - out.setWritePos(originalLength + sizeof(bool) + sizeof(size32_t) + compressedLen); - return; + compressor->open(newData, newSize); + if (compressor->write(src, len)==len) + { + compressor->close(); + size32_t compressedLen = compressor->buflen(); + out.setWritePos(originalLength + sizeof(byte)); + out.append(compressedLen); + out.setWritePos(originalLength + sizeof(byte) + sizeof(size32_t) + compressedLen); + return; + } + } + catch (IException *E) + { + E->Release(); } + // failed to compress... + out.setWritePos(originalLength); } - - // all or don't compress - out.setWritePos(originalLength); - appendToBuffer(out, len, src); -} - -void decompressToBuffer(MemoryBuffer & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.reserve(outSize); - expander->expand(buff); + out.append((byte) COMPRESS_METHOD_NONE); + out.append(len); + out.append(len, src); } - -void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in) +void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options) { - bool compressed; size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToBuffer(out, in.readDirect(srcLen)); - else + unsigned char _method; + in.read(_method).read(srcLen); + CompressionMethod method = (CompressionMethod) _method; + if (method==COMPRESS_METHOD_NONE) out.append(srcLen, in.readDirect(srcLen)); -} - -void decompressToAttr(MemoryAttr & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.allocate(outSize); - expander->expand(buff); -} - -void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in) -{ - bool compressed; - size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToAttr(out, in.readDirect(srcLen)); else - out.set(srcLen, in.readDirect(srcLen)); + { + if (method==COMPRESS_METHOD_LZWLEGACY) + method = COMPRESS_METHOD_LZW; // Back compatibilty + ICompressHandler *handler = queryCompressHandler(method); + if (!handler) + { + VStringBuffer s("Unknown decompression method %x required in decompressToBuffer", (byte) method); + throw makeStringException(0, s.str()); + } + Owned expander = handler->getExpander(options); + unsigned outSize = expander->init(in.readDirect(srcLen)); + void * buff = out.reserve(outSize); + expander->expand(buff); + } } - - /* Simple Diff compression format is @@ -2604,6 +2598,8 @@ class CAESCompressor : implements ICompressor, public CInterface else outbuf = outattr.allocate(blksize); outBufMb = NULL; + if (blksize <= AES_PADDING_SIZE+sizeof(size32_t)) + throw makeStringException(0, "CAESCompressor: target buffer too small"); size32_t subsz = blksize-AES_PADDING_SIZE-sizeof(size32_t); comp->open(compattr.reserveTruncate(subsz),subsz); } @@ -2788,44 +2784,88 @@ IPropertyTree *getBlockedFileDetails(IFile *file) return NULL; } -class CCompressHandlerArray : public IArrayOf +class CCompressHandlerArray { + IArrayOf registered; // Owns the relevant handler objects + ICompressHandler *byMethod[COMPRESS_METHOD_LAST] = { nullptr }; + ICompressHandler *AESbyMethod[COMPRESS_METHOD_LAST] = { nullptr }; + public: ICompressHandler *lookup(const char *type) const { - ForEachItemIn(h, *this) + ForEachItemIn(h, registered) { - ICompressHandler &handler = item(h); + ICompressHandler &handler = registered.item(h); if (0 == stricmp(type, handler.queryType())) return &handler; } return NULL; } + ICompressHandler *lookup(CompressionMethod method) const + { + if ((method & ~COMPRESS_METHOD_AES) >= COMPRESS_METHOD_LAST) + return nullptr; + else if (method & COMPRESS_METHOD_AES) + return AESbyMethod[method & ~COMPRESS_METHOD_AES]; + else + return byMethod[method]; + } + ICompressHandlerIterator *getIterator() + { + return new ArrayIIteratorOf, ICompressHandler, ICompressHandlerIterator>(registered); + } + bool addCompressor(ICompressHandler *handler) + { + CompressionMethod method = handler->queryMethod(); + if (lookup(method)) + { + handler->Release(); + return false; // already registered + } + registered.append(* handler); + if ((method & ~COMPRESS_METHOD_AES) < COMPRESS_METHOD_LAST) + { + if (method & COMPRESS_METHOD_AES) + AESbyMethod[method & ~COMPRESS_METHOD_AES] = handler; + else + byMethod[method] = handler; + } + return true; + } + bool removeCompressor(ICompressHandler *handler) + { + CompressionMethod method = handler->queryMethod(); + if (registered.zap(* handler)) + { + if ((method & ~COMPRESS_METHOD_AES) < COMPRESS_METHOD_LAST) + { + if (method & COMPRESS_METHOD_AES) + AESbyMethod[method & ~COMPRESS_METHOD_AES] = handler; + else + byMethod[method] = handler; + } + return true; + } + else + return false; + } } compressors; typedef IIteratorOf ICompressHandlerIterator; ICompressHandlerIterator *getCompressHandlerIterator() { - return new ArrayIIteratorOf, ICompressHandler, ICompressHandlerIterator>(compressors); + return compressors.getIterator(); } - - bool addCompressorHandler(ICompressHandler *handler) { - if (compressors.lookup(handler->queryType())) - { - handler->Release(); - return false; // already registered - } - compressors.append(* handler); - return true; + return compressors.addCompressor(handler); } bool removeCompressorHandler(ICompressHandler *handler) { - return compressors.zap(* handler); + return compressors.removeCompressor(handler); } Linked defaultCompressor; @@ -2834,38 +2874,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) { class CCompressHandlerBase : implements ICompressHandler, public CInterface { - StringAttr type; public: IMPLEMENT_IINTERFACE; - CCompressHandlerBase(const char *_type) : type(_type) { } - // ICompressHandler - virtual const char *queryType() const { return type; } }; class CFLZCompressHandler : public CCompressHandlerBase { public: - CFLZCompressHandler() : CCompressHandlerBase("FLZ") { } + virtual const char *queryType() const { return "FLZ"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_FASTLZ; } virtual ICompressor *getCompressor(const char *options) { return createFastLZCompressor(); } virtual IExpander *getExpander(const char *options) { return createFastLZExpander(); } }; class CLZ4CompressHandler : public CCompressHandlerBase { public: - CLZ4CompressHandler() : CCompressHandlerBase("LZ4") { } + virtual const char *queryType() const { return "LZ4"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(false); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CLZ4HCCompressHandler : public CCompressHandlerBase { public: - CLZ4HCCompressHandler() : CCompressHandlerBase("LZ4HC") { } + virtual const char *queryType() const { return "LZ4HC"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4HC; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(true); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CAESCompressHandler : public CCompressHandlerBase { public: - CAESCompressHandler() : CCompressHandlerBase("AES") { } + virtual const char *queryType() const { return "AES"; } + virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZW); } virtual ICompressor *getCompressor(const char *options) { assertex(options); @@ -2880,20 +2920,22 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) class CDiffCompressHandler : public CCompressHandlerBase { public: - CDiffCompressHandler() : CCompressHandlerBase("DIFF") { } + virtual const char *queryType() const { return "DIFF"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; } virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); } }; class CLZWCompressHandler : public CCompressHandlerBase { public: - CLZWCompressHandler() : CCompressHandlerBase("LZW") { } + virtual const char *queryType() const { return "LZW"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZW; } virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(true); } virtual IExpander *getExpander(const char *options) { return createLZWExpander(true); } }; + addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CAESCompressHandler()); addCompressorHandler(new CDiffCompressHandler()); - addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CFLZCompressHandler()); addCompressorHandler(new CLZ4HCCompressHandler()); ICompressHandler *lz4Compressor = new CLZ4CompressHandler(); @@ -2906,6 +2948,10 @@ ICompressHandler *queryCompressHandler(const char *type) { return compressors.lookup(type); } +ICompressHandler *queryCompressHandler(CompressionMethod method) +{ + return compressors.lookup(method); +} void setDefaultCompressor(const char *type) { diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp index 391bee8732b..51f7a70a097 100644 --- a/system/jlib/jlzw.hpp +++ b/system/jlib/jlzw.hpp @@ -24,6 +24,25 @@ #include "jfile.hpp" #include +enum CompressionMethod +{ + //These values are persisted - they should not be changed + COMPRESS_METHOD_NONE, + COMPRESS_METHOD_ROWDIF, + COMPRESS_METHOD_LZW, + COMPRESS_METHOD_FASTLZ, + COMPRESS_METHOD_LZMA, + COMPRESS_METHOD_LZ4, + COMPRESS_METHOD_LZ4HC, + COMPRESS_METHOD_RANDROW, + COMPRESS_METHOD_LAST, + + + COMPRESS_METHOD_AES = 0x80, + COMPRESS_METHOD_LZWLEGACY = 1, // Matches value of boolean 'true' used to indicate LZW compression by legacy compressToBuffer +}; + + interface jlib_decl ICompressor : public IInterface { virtual void open(MemoryBuffer &mb, size32_t initialSize=0)=0; // variable internally sized buffer @@ -87,21 +106,9 @@ extern jlib_decl ICompressor *createRandRDiffCompressor(); // similar to RDiffCo extern jlib_decl IRandRowExpander *createRandRDiffExpander(); // NB only supports fixed row size -//Some helper functions to make it easy to compress/decompress to memorybuffers. -extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in); -extern jlib_decl void decompressToAttr(MemoryAttr & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in); -extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src); //format as failed compression - - -#define COMPRESS_METHOD_ROWDIF 1 -#define COMPRESS_METHOD_LZW 2 -#define COMPRESS_METHOD_FASTLZ 3 -#define COMPRESS_METHOD_LZMA 4 -#define COMPRESS_METHOD_LZ4 5 -#define COMPRESS_METHOD_LZ4HC 6 +// Helper functions to make it easy to compress/decompress to memorybuffers. +extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method=COMPRESS_METHOD_LZW, const char *options=nullptr); +extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options=nullptr); interface ICompressedFileIO: extends IFileIO { @@ -135,6 +142,7 @@ extern jlib_decl IPropertyTree *getBlockedFileDetails(IFile *file); interface ICompressHandler : extends IInterface { virtual const char *queryType() const = 0; + virtual CompressionMethod queryMethod() const = 0; virtual ICompressor *getCompressor(const char *options=NULL) = 0; virtual IExpander *getExpander(const char *options=NULL) = 0; }; @@ -142,6 +150,7 @@ typedef IIteratorOf ICompressHandlerIterator; extern jlib_decl ICompressHandlerIterator *getCompressHandlerIterator(); extern jlib_decl void setDefaultCompressor(const char *type); extern jlib_decl ICompressHandler *queryCompressHandler(const char *type); +extern jlib_decl ICompressHandler *queryCompressHandler(CompressionMethod method); extern jlib_decl ICompressHandler *queryDefaultCompressHandler(); extern jlib_decl bool addCompressorHandler(ICompressHandler *handler); // returns true if added, false if already registered extern jlib_decl bool removeCompressorHandler(ICompressHandler *handler); // returns true if present and removed diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 74eb554893f..1a93caaa5d9 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -2642,15 +2642,15 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture CPPUNIT_TEST(test); CPPUNIT_TEST_SUITE_END(); + static constexpr size32_t sz = 100*0x100000; // 100MB + enum CompressOpt { RowCompress, AllRowCompress, BlockCompress, CompressToBuffer }; public: void test() { try { - size32_t sz = 100*0x100000; // 100MB MemoryBuffer src; src.ensureCapacity(sz); - MemoryBuffer compressed; const char *aesKey = "012345678901234567890123"; Owned iter = getCompressHandlerIterator(); @@ -2680,19 +2680,66 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture } } - DBGLOG("Algorithm || Compression Time (ms) || Decompression Time (ms) || Compression Ratio"); + DBGLOG("Algorithm(options) || Comp(ms) || Deco(ms) || 200MB/s (w,r) || 1GB/s (w,r) || 5GB/s (w,r) || Ratio [cLen]"); + DBGLOG(" || || || 2Gb/s || 10Gb/s || 50Gb/s ||"); + unsigned time200MBs = transferTimeMs(sz, 200000000); + unsigned time1GBs = transferTimeMs(sz, 1000000000); + unsigned time5GBs = transferTimeMs(sz, 5000000000); + DBGLOG("%19s || %8u || %8u || %4u(%4u,%4u) || %4u(%4u,%4u) || %4u(%4u,%4u) || %5.2f [%u]", "uncompressed", 0, 0, + time200MBs, time200MBs, time200MBs, time1GBs, time1GBs, time1GBs, time5GBs, time5GBs, time5GBs, 1.0, sz); ForEach(*iter) { - compressed.clear(); ICompressHandler &handler = iter->query(); - Owned compressor = handler.getCompressor(streq("AES", handler.queryType()) ? aesKey: nullptr); + const char * type = handler.queryType(); + //Ignore unusual compressors with no expanders... + if (strieq(type, "randrow")) + continue; + const char * options = streq("AES", handler.queryType()) ? aesKey: ""; + if (streq(type, "LZ4HC")) + { + testCompressor(handler, "hclevel=3", rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, "hclevel=4", rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, "hclevel=5", rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, "hclevel=6", rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, "hclevel=8", rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, "hclevel=10", rowSz, src.length(), src.bytes(), RowCompress); + } + testCompressor(handler, options, rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, options, rowSz, src.length(), src.bytes(), CompressToBuffer); + if (streq(type, "LZ4")) + { + testCompressor(handler, "allrow", rowSz, src.length(), src.bytes(), AllRowCompress); // block doesn't affect the compressor, just tracing + testCompressor(handler, "block", rowSz, src.length(), src.bytes(), BlockCompress); // block doesn't affect the compressor, just tracing + } + } + } + catch (IException *e) + { + EXCLOG(e, nullptr); + throw; + } + } - CCycleTimer timer; + unsigned transferTimeMs(__int64 size, __int64 bytesPerSecond) + { + return (unsigned)((size * 1000) / bytesPerSecond); + } + + void testCompressor(ICompressHandler &handler, const char * options, size32_t rowSz, size32_t srcLen, const byte * src, CompressOpt opt) + { + Owned compressor = handler.getCompressor(options); + + MemoryBuffer compressed; + CCycleTimer timer; + const byte * ptr = src; + switch (opt) + { + case RowCompress: + { compressor->open(compressed, sz); compressor->startblock(); - const byte *ptr = src.bytes(); - const byte *ptrEnd = ptr + src.length(); + const byte *ptrEnd = ptr + srcLen; while (ptr != ptrEnd) { compressor->write(ptr, rowSz); @@ -2700,30 +2747,77 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture } compressor->commitblock(); compressor->close(); - cycle_t compressCycles = timer.elapsedCycles(); - - Owned expander = handler.getExpander(streq("AES", handler.queryType()) ? aesKey: nullptr); - - timer.reset(); - size32_t required = expander->init(compressed.bytes()); - MemoryBuffer tgt(required); - expander->expand(tgt.bufferBase()); - tgt.setWritePos(required); - cycle_t decompressCycles = timer.elapsedCycles(); - - float ratio = (float)(src.length()) / compressed.length(); - - DBGLOG("%9s || %21u || %23u || %17.2f [ %u, %u ]", handler.queryType(), (unsigned)cycle_to_millisec(compressCycles), (unsigned)cycle_to_millisec(decompressCycles), ratio, src.length(), compressed.length()); - - CPPUNIT_ASSERT(tgt.length() >= sz); - CPPUNIT_ASSERT(0 == memcmp(src.bufferBase(), tgt.bufferBase(), sz)); - } + break; + } + case AllRowCompress: + { + compressor->open(compressed, sz); + compressor->startblock(); + compressor->write(ptr, sz); + compressor->commitblock(); + compressor->close(); + break; + } + case BlockCompress: + { + void * target = compressed.reserve(sz); + unsigned written = compressor->compressBlock(sz, target, srcLen, ptr); + compressed.setLength(written); + break; + } + case CompressToBuffer: + { + compressToBuffer(compressed, srcLen, ptr, handler.queryMethod(), options); + break; + } } - catch (IException *e) + + cycle_t compressCycles = timer.elapsedCycles(); + Owned expander = handler.getExpander(options); + MemoryBuffer tgt; + timer.reset(); + if (opt==CompressToBuffer) { - EXCLOG(e, nullptr); - throw; + decompressToBuffer(tgt, compressed, options); } + else + { + size32_t required = expander->init(compressed.bytes()); + tgt.reserveTruncate(required); + expander->expand(tgt.bufferBase()); + tgt.setWritePos(required); + } + cycle_t decompressCycles = timer.elapsedCycles(); + + float ratio = (float)(srcLen) / compressed.length(); + + StringBuffer name(handler.queryType()); + if (opt == CompressToBuffer) + name.append("-c2b"); + if (options && *options) + name.append("-").append(options); + + + if (name.length() > 19) + name.setLength(19); + + unsigned compressTime = (unsigned)cycle_to_millisec(compressCycles); + unsigned decompressTime = (unsigned)cycle_to_millisec(decompressCycles); + unsigned compressedTime = compressTime + decompressTime; + unsigned copyTime200MBs = transferTimeMs(compressed.length(), 200000000); + unsigned copyTime1GBs = transferTimeMs(compressed.length(), 1000000000); + unsigned copyTime5GBs = transferTimeMs(compressed.length(), 5000000000); + unsigned time200MBs = copyTime200MBs + compressedTime; + unsigned time1GBs = copyTime1GBs + compressedTime; + unsigned time5GBs = copyTime5GBs + compressedTime; + DBGLOG("%19s || %8u || %8u || %4u(%4u,%4u) || %4u(%4u,%4u) || %4u(%4u,%4u) || %5.2f [%u]", name.str(), compressTime, decompressTime, + time200MBs, copyTime200MBs + compressTime, copyTime200MBs + decompressTime, + time1GBs, copyTime1GBs + compressTime, copyTime1GBs + decompressTime, + time5GBs, copyTime5GBs + compressTime, copyTime5GBs + decompressTime, + ratio, compressed.length()); + + CPPUNIT_ASSERT(tgt.length() >= sz); + CPPUNIT_ASSERT(0 == memcmp(src, tgt.bufferBase(), sz)); } }; diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index 21df821f322..6ec19980649 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -928,4 +928,79 @@ CPPUNIT_TEST_SUITE_REGISTRATION( PipeRunTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( PipeRunTest, "PipeRunTest" ); #endif +#include "jlzw.hpp" +class compressToBufferTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE( compressToBufferTest ); + CPPUNIT_TEST(testRun); + CPPUNIT_TEST_SUITE_END(); + + void testOne(unsigned len, CompressionMethod method, const char *options=nullptr) + { + constexpr const char *in = + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello"; + assertex(len <= strlen(in)); + MemoryBuffer compressed; + compressToBuffer(compressed, len, in, method, options); + + if (method != COMPRESS_METHOD_NONE && len >= 32 && compressed.length() == len+5) + DBGLOG("compressToBuffer %x size %u did not compress", (byte) method, len); + else + DBGLOG("compressToBuffer %x size %u compressed to %u", (byte) method, len, compressed.length()); + CPPUNIT_ASSERT(compressed.length() <= len+5); + MemoryBuffer out; + decompressToBuffer(out, compressed, options); + CPPUNIT_ASSERT(out.length() == len); + CPPUNIT_ASSERT(memcmp(out.bytes(), in, len) == 0); + } + + void testSome(unsigned len) + { + testOne(len, COMPRESS_METHOD_NONE); + testOne(len, COMPRESS_METHOD_LZW); + testOne(len, COMPRESS_METHOD_LZ4); + testOne(len, (CompressionMethod) (COMPRESS_METHOD_LZW|COMPRESS_METHOD_AES), "0123456789abcdef"); + } + + void testRun() + { + testSome(0); + testSome(1); + testSome(16); + testSome(32); + testSome(200); + testSome(1000); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( compressToBufferTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( compressToBufferTest, "CompressToBufferTest" ); + + #endif // _USE_CPPUNIT