From 797a1195c24a40e44f9cef6a257fb3e69660d2b4 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 1 Nov 2023 14:55:17 +0000 Subject: [PATCH 1/5] Refactor compressToBuffer to support different compression methods Signed-off-by: Richard Chapman --- common/dllserver/thorplugin.cpp | 5 +- system/jlib/jlzw.cpp | 158 +++++++++++++++----------------- system/jlib/jlzw.hpp | 12 +-- testing/unittests/jlibtests.cpp | 84 +++++++++++------ testing/unittests/unittests.cpp | 42 +++++++++ 5 files changed, 177 insertions(+), 124 deletions(-) diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp index 5db7bfce4e7..0fbf4079305 100644 --- a/common/dllserver/thorplugin.cpp +++ b/common/dllserver/thorplugin.cpp @@ -679,10 +679,7 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress) { mb.append((byte)0x80).append(resourceHeaderVersion); - if (compress) - compressToBuffer(mb, len, data); - else - appendToBuffer(mb, len, data); + compressToBuffer(mb, len, data, compress ? COMPRESS_METHOD_LZW : COMPRESS_METHOD_NONE); } extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data) diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 2dbf48aca62..c541aaa6312 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -758,82 +758,61 @@ size32_t RLEExpand(void *dst,const void *src,size32_t expsize) return (size32_t)(in-(const byte *)src); } -void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src) -{ - out.append(false); - out.append(len); - out.append(len, src); -} - -void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src) -{ - unsigned originalLength = out.length(); - out.append(true); - out.append((size32_t)0); - - if (len >= 32) - { - size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ... - Owned compressor = createLZWCompressor(); - void *newData = out.reserve(newSize); - compressor->open(newData, newSize); - if (compressor->write(src, len)==len) +void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method, const char *options) +{ + if (method != COMPRESS_METHOD_NONE) + { + unsigned originalLength = out.length(); + // For back-compatibility, we always store COMPRESS_METHOD_LZW as 1 as earlier versions stored a boolean here + // rather than an enum + // This means that compressToBuffer/decompressToBuffer cannot bs used for rowdiff compression - this is not likely to be an issue + // Alternative would be a separate enum for compressToBuffer formats, but that seems more likely to cause confusion + out.append((byte) (method == COMPRESS_METHOD_LZW ? COMPRESS_METHOD_LZWLEGACY : method)); + out.append((size32_t)0); + if (len >= 32) { - compressor->close(); - size32_t compressedLen = compressor->buflen(); - out.setWritePos(originalLength + sizeof(bool)); - out.append(compressedLen); - out.setWritePos(originalLength + sizeof(bool) + sizeof(size32_t) + compressedLen); - return; + size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ... + Owned compressor = queryCompressHandler(method)->getCompressor(options); + void *newData = out.reserve(newSize); + compressor->open(newData, newSize); + if (compressor->write(src, len)==len) + { + compressor->close(); + size32_t compressedLen = compressor->buflen(); + out.setWritePos(originalLength + sizeof(bool)); + out.append(compressedLen); + out.setWritePos(originalLength + sizeof(bool) + sizeof(size32_t) + compressedLen); + return; + } } + + // all or don't compress + out.setWritePos(originalLength); } - - // all or don't compress - out.setWritePos(originalLength); - appendToBuffer(out, len, src); -} - -void decompressToBuffer(MemoryBuffer & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.reserve(outSize); - expander->expand(buff); + out.append((byte) COMPRESS_METHOD_NONE); + out.append(len); + out.append(len, src); } - -void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in) +void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options) { - bool compressed; size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToBuffer(out, in.readDirect(srcLen)); - else + unsigned char _method; + in.read(_method).read(srcLen); + CompressionMethod method = (CompressionMethod) _method; + if (method==COMPRESS_METHOD_NONE) out.append(srcLen, in.readDirect(srcLen)); -} - -void decompressToAttr(MemoryAttr & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.allocate(outSize); - expander->expand(buff); -} - -void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in) -{ - bool compressed; - size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToAttr(out, in.readDirect(srcLen)); else - out.set(srcLen, in.readDirect(srcLen)); + { + if (method==COMPRESS_METHOD_LZWLEGACY) + method = COMPRESS_METHOD_LZW; // Back compatibilty + Owned expander = queryCompressHandler(method)->getExpander(options); + unsigned outSize = expander->init(in.readDirect(srcLen)); + void * buff = out.reserve(outSize); + expander->expand(buff); + } } - - /* Simple Diff compression format is @@ -2875,6 +2854,18 @@ class CCompressHandlerArray : public IArrayOf } return NULL; } + ICompressHandler *lookup(CompressionMethod method) const + { + // MORE - should probably use an array, cache last lookup, or something... + // This is called quite a lot now + ForEachItemIn(h, *this) + { + ICompressHandler &handler = item(h); + if (method == handler.queryMethod()) + return &handler; + } + return NULL; + } } compressors; typedef IIteratorOf ICompressHandlerIterator; @@ -2884,11 +2875,9 @@ ICompressHandlerIterator *getCompressHandlerIterator() return new ArrayIIteratorOf, ICompressHandler, ICompressHandlerIterator>(compressors); } - - bool addCompressorHandler(ICompressHandler *handler) { - if (compressors.lookup(handler->queryType())) + if (compressors.lookup(handler->queryMethod())) { handler->Release(); return false; // already registered @@ -2908,38 +2897,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) { class CCompressHandlerBase : implements ICompressHandler, public CInterface { - StringAttr type; public: IMPLEMENT_IINTERFACE; - CCompressHandlerBase(const char *_type) : type(_type) { } - // ICompressHandler - virtual const char *queryType() const { return type; } }; class CFLZCompressHandler : public CCompressHandlerBase { public: - CFLZCompressHandler() : CCompressHandlerBase("FLZ") { } + virtual const char *queryType() const { return "FLZ"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_FASTLZ; } virtual ICompressor *getCompressor(const char *options) { return createFastLZCompressor(); } virtual IExpander *getExpander(const char *options) { return createFastLZExpander(); } }; class CLZ4CompressHandler : public CCompressHandlerBase { public: - CLZ4CompressHandler() : CCompressHandlerBase("LZ4") { } + virtual const char *queryType() const { return "LZ4"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, false); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CLZ4HCCompressHandler : public CCompressHandlerBase { public: - CLZ4HCCompressHandler() : CCompressHandlerBase("LZ4HC") { } + virtual const char *queryType() const { return "LZ4HC"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4HC; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, true); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CAESCompressHandler : public CCompressHandlerBase { public: - CAESCompressHandler() : CCompressHandlerBase("AES") { } + virtual const char *queryType() const { return "AES"; } + virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZW); } virtual ICompressor *getCompressor(const char *options) { assertex(options); @@ -2954,34 +2943,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) class CDiffCompressHandler : public CCompressHandlerBase { public: - CDiffCompressHandler() : CCompressHandlerBase("DIFF") { } + virtual const char *queryType() const { return "DIFF"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; } virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); } }; class CRDiffCompressHandler : public CCompressHandlerBase { public: - CRDiffCompressHandler() : CCompressHandlerBase("RDIFF") { } + virtual const char *queryType() const { return "RDIFF"; } // Synonym for DIFF + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; } virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); } }; class CRandRDiffCompressHandler : public CCompressHandlerBase { public: - CRandRDiffCompressHandler() : CCompressHandlerBase("RANDROW") { } + virtual const char *queryType() const { return "RANDROW"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_RANDROW; } virtual ICompressor *getCompressor(const char *options) { return createRandRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { UNIMPLEMENTED; } // Expander has a different interface }; class CLZWCompressHandler : public CCompressHandlerBase { public: - CLZWCompressHandler() : CCompressHandlerBase("LZW") { } + virtual const char *queryType() const { return "LZW"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZW; } virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(true); } virtual IExpander *getExpander(const char *options) { return createLZWExpander(true); } }; + addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CAESCompressHandler()); addCompressorHandler(new CDiffCompressHandler()); - addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CRDiffCompressHandler()); addCompressorHandler(new CRandRDiffCompressHandler()); addCompressorHandler(new CFLZCompressHandler()); @@ -2999,8 +2992,7 @@ ICompressHandler *queryCompressHandler(const char *type) ICompressHandler *queryCompressHandler(CompressionMethod method) { - //Could be more efficient, but doesn't matter - return compressors.lookup(translateFromCompMethod(method)); + return compressors.lookup(method); } void setDefaultCompressor(const char *type) diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp index 1d7c7196390..cedcfc2187e 100644 --- a/system/jlib/jlzw.hpp +++ b/system/jlib/jlzw.hpp @@ -38,6 +38,7 @@ enum CompressionMethod COMPRESS_METHOD_AES = 0x80, + COMPRESS_METHOD_LZWLEGACY = 1, // Matches value of boolean 'true' used to indicate LZW compression by legacy compressToBuffer }; @@ -117,13 +118,9 @@ extern jlib_decl ICompressor *createRandRDiffCompressor(); // similar to RDiffCo extern jlib_decl IRandRowExpander *createRandRDiffExpander(); // NB only supports fixed row size -//Some helper functions to make it easy to compress/decompress to memorybuffers. -extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in); -extern jlib_decl void decompressToAttr(MemoryAttr & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in); -extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src); //format as failed compression +// Helper functions to make it easy to compress/decompress to memorybuffers. +extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method=COMPRESS_METHOD_LZW, const char *options=nullptr); +extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options=nullptr); interface ICompressedFileIO: extends IFileIO @@ -158,6 +155,7 @@ extern jlib_decl IPropertyTree *getBlockedFileDetails(IFile *file); interface ICompressHandler : extends IInterface { virtual const char *queryType() const = 0; + virtual CompressionMethod queryMethod() const = 0; virtual ICompressor *getCompressor(const char *options=NULL) = 0; virtual IExpander *getExpander(const char *options=NULL) = 0; }; diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 0f3a99b29fe..e30152e0f3b 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3085,11 +3085,14 @@ CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(JlibIOTest, "JlibIOTest"); class JlibCompressionTestsStress : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(JlibCompressionTestsStress); + CPPUNIT_TEST(testc2b); CPPUNIT_TEST(test); CPPUNIT_TEST_SUITE_END(); public: - void test() + void test() { dotest(false); } + void testc2b() { dotest(true); } + void dotest(bool c2b) { try { @@ -3135,37 +3138,58 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture //Ignore unusual compressors with no expanders... if (strieq(handler.queryType(), "randrow")) continue; - Owned compressor = handler.getCompressor(streq("AES", handler.queryType()) ? aesKey: nullptr); - - CCycleTimer timer; - compressor->open(compressed, sz); - compressor->startblock(); - const byte *ptr = src.bytes(); - const byte *ptrEnd = ptr + src.length(); - while (ptr != ptrEnd) + try + { + cycle_t compressCycles; + cycle_t decompressCycles; + MemoryBuffer tgt; + if (c2b) + { + CCycleTimer timer; + compressToBuffer(compressed, src.length(), src.bytes(), handler.queryMethod(), streq("AES", handler.queryType()) ? aesKey: nullptr); + compressCycles = timer.elapsedCycles(); + timer.reset(); + decompressToBuffer(tgt, compressed, streq("AES", handler.queryType()) ? aesKey: nullptr); + decompressCycles = timer.elapsedCycles(); + } + else + { + Owned compressor = handler.getCompressor(streq("AES", handler.queryType()) ? aesKey: nullptr); + + CCycleTimer timer; + compressor->open(compressed, sz); + compressor->startblock(); + const byte *ptr = src.bytes(); + const byte *ptrEnd = ptr + src.length(); + while (ptr != ptrEnd) + { + compressor->write(ptr, rowSz); + ptr += rowSz; + } + compressor->commitblock(); + compressor->close(); + compressCycles = timer.elapsedCycles(); + + Owned expander = handler.getExpander(streq("AES", handler.queryType()) ? aesKey: nullptr); + + timer.reset(); + size32_t required = expander->init(compressed.bytes()); + tgt.ensureCapacity(required); + expander->expand(tgt.bufferBase()); + tgt.setWritePos(required); + decompressCycles = timer.elapsedCycles(); + } + float ratio = (float)(src.length()) / compressed.length(); + DBGLOG("%9s || %21u || %23u || %17.2f [ %u, %u ]", handler.queryType(), (unsigned)cycle_to_millisec(compressCycles), (unsigned)cycle_to_millisec(decompressCycles), ratio, src.length(), compressed.length()); + CPPUNIT_ASSERT(tgt.length() >= sz); + CPPUNIT_ASSERT(0 == memcmp(src.bufferBase(), tgt.bufferBase(), sz)); + } + catch (IException *E) { - compressor->write(ptr, rowSz); - ptr += rowSz; + StringBuffer str; + DBGLOG("%9s threw exception %s", handler.queryType(), E->errorMessage(str).str()); + E->Release(); } - compressor->commitblock(); - compressor->close(); - cycle_t compressCycles = timer.elapsedCycles(); - - Owned expander = handler.getExpander(streq("AES", handler.queryType()) ? aesKey: nullptr); - - timer.reset(); - size32_t required = expander->init(compressed.bytes()); - MemoryBuffer tgt(required); - expander->expand(tgt.bufferBase()); - tgt.setWritePos(required); - cycle_t decompressCycles = timer.elapsedCycles(); - - float ratio = (float)(src.length()) / compressed.length(); - - DBGLOG("%9s || %21u || %23u || %17.2f [ %u, %u ]", handler.queryType(), (unsigned)cycle_to_millisec(compressCycles), (unsigned)cycle_to_millisec(decompressCycles), ratio, src.length(), compressed.length()); - - CPPUNIT_ASSERT(tgt.length() >= sz); - CPPUNIT_ASSERT(0 == memcmp(src.bufferBase(), tgt.bufferBase(), sz)); } } catch (IException *e) diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index a4c7e7e8ca6..a8b0545acec 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -1052,5 +1052,47 @@ class RelaxedAtomicTimingTest : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_REGISTRATION( RelaxedAtomicTimingTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( RelaxedAtomicTimingTest, "RelaxedAtomicTimingTest" ); +#include "jlzw.hpp" +class compressToBufferTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE( compressToBufferTest ); + CPPUNIT_TEST(testRun); + CPPUNIT_TEST_SUITE_END(); + + void testRun() + { + MemoryBuffer x; + compressToBuffer(x, 251, + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + ); + for (unsigned i = 0; i < x.length(); i++) + printf("%02x ", x.toByteArray()[i]); + printf("\n"); + * (byte *) x.toByteArray() = 2; + for (unsigned i = 0; i < x.length(); i++) + printf("%02x ", x.toByteArray()[i]); + printf("\n"); + try + { + MemoryBuffer out; + decompressToBuffer(out, x); + printf("%s\n", out.toByteArray()); + } + catch(IException *E) + { + StringBuffer s; + printf("Exception %s\n", E->errorMessage(s).str()); + ::Release(E); + } + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( compressToBufferTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( compressToBufferTest, "CompressToBufferTest" ); + #endif // _USE_CPPUNIT From 8f3e07990a85b7bc4232d993ee4f314c056c3caf Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 8 Nov 2023 15:29:41 +0000 Subject: [PATCH 2/5] Add AESLZ4 and AESLZ4HC modes Signed-off-by: Richard Chapman --- system/jlib/jlzw.cpp | 56 ++++++++++++++++++++++++++------- testing/unittests/jlibtests.cpp | 8 ++--- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index c541aaa6312..2fd0258fd53 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -2608,10 +2608,10 @@ ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bo #define AES_PADDING_SIZE 32 - class CAESCompressor : implements ICompressor, public CInterface { - Owned comp; // base compressor +private: + Owned comp; MemoryBuffer compattr; // compressed buffer MemoryAttr outattr; // compressed and encrypted (if outblk NULL) void *outbuf; // dest @@ -2620,13 +2620,11 @@ class CAESCompressor : implements ICompressor, public CInterface size32_t originalMax = 0; MemoryAttr key; MemoryBuffer *outBufMb; - public: IMPLEMENT_IINTERFACE; - CAESCompressor(const void *_key, unsigned _keylen) - : key(_keylen,_key) + CAESCompressor(const void *_key, unsigned _keylen, ICompressor *_comp) + : comp(_comp), key(_keylen,_key) { - comp.setown(createLZWCompressor(true)); outlen = 0; outmax = 0; outBufMb = NULL; @@ -2726,10 +2724,10 @@ class CAESExpander : implements CExpanderBase MemoryBuffer compbuf; MemoryAttr key; public: - CAESExpander(const void *_key, unsigned _keylen) + CAESExpander(const void *_key, unsigned _keylen, IExpander *_exp) : key(_keylen,_key) { - exp.setown(createLZWExpander(true)); + exp.setown(_exp); } size32_t init(const void *blk) { @@ -2759,11 +2757,11 @@ class CAESExpander : implements CExpanderBase ICompressor *createAESCompressor(const void *key, unsigned keylen) { - return new CAESCompressor(key,keylen); + return new CAESCompressor(key,keylen,createLZWCompressor(true)); } IExpander *createAESExpander(const void *key, unsigned keylen) { - return new CAESExpander(key,keylen); + return new CAESExpander(key,keylen,createLZWExpander(true)); } #define ROTATE_BYTE_LEFT(x, n) (((x) << (n)) | ((x) >> (8 - (n)))) @@ -2790,14 +2788,14 @@ ICompressor *createAESCompressor256(size32_t len, const void *key) { byte k[32]; padKey32(k,len,(const byte *)key); - return new CAESCompressor(k,32); + return new CAESCompressor(k,32,createLZWCompressor(true)); } IExpander *createAESExpander256(size32_t len, const void *key) { byte k[32]; padKey32(k,len,(const byte *)key); - return new CAESExpander(k,32); + return new CAESExpander(k,32,createLZWExpander(true)); } @@ -2940,6 +2938,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) return createAESExpander(options, strlen(options)); } }; + class CAESLZ4CompressHandler : public CCompressHandlerBase + { + public: + virtual const char *queryType() const { return "AESLZ4"; } + virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZ4); } + virtual ICompressor *getCompressor(const char *options) + { + assertex(options); + return new CAESCompressor(options, strlen(options),createLZ4Compressor(nullptr)); + } + virtual IExpander *getExpander(const char *options) + { + assertex(options); + return new CAESExpander(options, strlen(options), createLZ4Expander()); + } + }; + class CAESLZ4HCCompressHandler : public CCompressHandlerBase + { + public: + virtual const char *queryType() const { return "AESLZ4HC"; } + virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZ4HC); } + virtual ICompressor *getCompressor(const char *options) + { + assertex(options); + return new CAESCompressor(options, strlen(options),createLZ4Compressor(nullptr, true)); + } + virtual IExpander *getExpander(const char *options) + { + assertex(options); + return new CAESExpander(options, strlen(options), createLZ4Expander()); + } + }; class CDiffCompressHandler : public CCompressHandlerBase { public: @@ -2974,6 +3004,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) }; addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CAESCompressHandler()); + addCompressorHandler(new CAESLZ4CompressHandler()); + addCompressorHandler(new CAESLZ4HCCompressHandler()); addCompressorHandler(new CDiffCompressHandler()); addCompressorHandler(new CRDiffCompressHandler()); addCompressorHandler(new CRandRDiffCompressHandler()); diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index e30152e0f3b..43a93fde30f 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3146,15 +3146,15 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture if (c2b) { CCycleTimer timer; - compressToBuffer(compressed, src.length(), src.bytes(), handler.queryMethod(), streq("AES", handler.queryType()) ? aesKey: nullptr); + compressToBuffer(compressed, src.length(), src.bytes(), handler.queryMethod(), handler.queryMethod() & COMPRESS_METHOD_AES ? aesKey: nullptr); compressCycles = timer.elapsedCycles(); timer.reset(); - decompressToBuffer(tgt, compressed, streq("AES", handler.queryType()) ? aesKey: nullptr); + decompressToBuffer(tgt, compressed, handler.queryMethod() & COMPRESS_METHOD_AES ? aesKey: nullptr); decompressCycles = timer.elapsedCycles(); } else { - Owned compressor = handler.getCompressor(streq("AES", handler.queryType()) ? aesKey: nullptr); + Owned compressor = handler.getCompressor(handler.queryMethod() & COMPRESS_METHOD_AES ? aesKey: nullptr); CCycleTimer timer; compressor->open(compressed, sz); @@ -3170,7 +3170,7 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture compressor->close(); compressCycles = timer.elapsedCycles(); - Owned expander = handler.getExpander(streq("AES", handler.queryType()) ? aesKey: nullptr); + Owned expander = handler.getExpander(handler.queryMethod() & COMPRESS_METHOD_AES ? aesKey: nullptr); timer.reset(); size32_t required = expander->init(compressed.bytes()); From 9bd7efd8643524c4afc2224399b12f85de0d18e1 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 8 Nov 2023 15:46:22 +0000 Subject: [PATCH 3/5] Added hc level options for AES front end Signed-off-by: Richard Chapman --- system/jlib/jlzw.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 2fd0258fd53..19c2b5f436a 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -2875,7 +2875,7 @@ ICompressHandlerIterator *getCompressHandlerIterator() bool addCompressorHandler(ICompressHandler *handler) { - if (compressors.lookup(handler->queryMethod())) + if (compressors.lookup(handler->queryType())) { handler->Release(); return false; // already registered @@ -2956,13 +2956,20 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) }; class CAESLZ4HCCompressHandler : public CCompressHandlerBase { + StringBuffer typestring; + StringBuffer levelstring; public: - virtual const char *queryType() const { return "AESLZ4HC"; } + CAESLZ4HCCompressHandler(unsigned level) + { + typestring.appendf("AESLZ4HC_%u", level); + levelstring.appendf("hclevel=%u", level); + } + virtual const char *queryType() const { return typestring.str(); } virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZ4HC); } virtual ICompressor *getCompressor(const char *options) { assertex(options); - return new CAESCompressor(options, strlen(options),createLZ4Compressor(nullptr, true)); + return new CAESCompressor(options, strlen(options),createLZ4Compressor(levelstring, true)); } virtual IExpander *getExpander(const char *options) { @@ -3005,7 +3012,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CAESCompressHandler()); addCompressorHandler(new CAESLZ4CompressHandler()); - addCompressorHandler(new CAESLZ4HCCompressHandler()); + addCompressorHandler(new CAESLZ4HCCompressHandler(3)); + addCompressorHandler(new CAESLZ4HCCompressHandler(9)); addCompressorHandler(new CDiffCompressHandler()); addCompressorHandler(new CRDiffCompressHandler()); addCompressorHandler(new CRandRDiffCompressHandler()); From 53e916ec2b001604c393711f590a49a9c85a4504 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 8 Nov 2023 16:23:09 +0000 Subject: [PATCH 4/5] WIP Signed-off-by: Richard Chapman --- system/jlib/jencrypt.cpp | 126 ++++++++++++++++++++++++++++++++ system/jlib/jencrypt.hpp | 1 + testing/unittests/jlibtests.cpp | 19 +++++ 3 files changed, 146 insertions(+) diff --git a/system/jlib/jencrypt.cpp b/system/jlib/jencrypt.cpp index a0020f5f008..9ba53cc4fe5 100644 --- a/system/jlib/jencrypt.cpp +++ b/system/jlib/jencrypt.cpp @@ -1874,3 +1874,129 @@ void decrypt(StringBuffer &ret, const char *in) } } + + +#include +#include +#include +#include + +void handleErrors(void) +{ + ERR_print_errors_fp(stderr); + abort(); + +} + +MemoryBuffer &aesEncrypt_ssl(const void *key, unsigned keylen, const void *plaintext, size_t plaintext_len, MemoryBuffer &output) +{ + assertex(keylen==32); + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + if (!ctx) + throw makeStringException(0, "Crap"); + unsigned char iv[16] = { 0 }; + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, (const unsigned char *) key, iv)) + throw makeStringException(0, "Crap"); + byte *ciphertext = (byte *) output.reserve(plaintext_len + 100); + int ciphertext_len = 0; + if(1 != EVP_EncryptUpdate(ctx, ciphertext, &ciphertext_len, (const unsigned char *) plaintext, plaintext_len)) + throw makeStringException(0, "Crap"); + if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + ciphertext_len, &ciphertext_len)) + throw makeStringException(0, "Crap"); + EVP_CIPHER_CTX_free(ctx); + output.setLength(ciphertext_len); + return output; +} + +int aesDecrypt_ssl(unsigned char *ciphertext, int ciphertext_len, unsigned char *key, + unsigned char *iv, unsigned char *plaintext) +{ + EVP_CIPHER_CTX *ctx; + + int len; + + int plaintext_len; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) + handleErrors(); + + /* + * Initialise the decryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits + */ + if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, key, iv)) + handleErrors(); + + /* + * Provide the message to be decrypted, and obtain the plaintext output. + * EVP_DecryptUpdate can be called multiple times if necessary. + */ + if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) + handleErrors(); + plaintext_len = len; + + /* + * Finalise the decryption. Further plaintext bytes may be written at + * this stage. + */ + if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) + handleErrors(); + plaintext_len += len; + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return plaintext_len; +} + +void xmain (void) +{ + /* + * Set up the key and iv. Do I need to say to not hard code these in a + * real application? :-) + */ + + /* A 256 bit key */ + unsigned char key[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, + 0x36, 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31 + }; + + /* A 128 bit IV */ + unsigned char iv[16] = { 0 }; + + /* Message to be encrypted */ + unsigned char *plaintext = (unsigned char *)"The quick brown fox jumps over the lazy dog"; + + MemoryBuffer ciphertext; + + /* Buffer for the decrypted text */ + unsigned char decryptedtext[128]; + + int decryptedtext_len, ciphertext_len; + + /* Encrypt the plaintext */ + aesEncrypt_ssl(key, 32, plaintext, strlen ((char *)plaintext), ciphertext); + ciphertext_len = ciphertext.length(); + + /* Do something useful with the ciphertext here */ + printf("Ciphertext is:\n"); + BIO_dump_fp (stdout, ciphertext.bytes(), ciphertext.length()); + + /* Decrypt the ciphertext */ + decryptedtext_len = aesDecrypt_ssl((unsigned char *) ciphertext.bytes(), ciphertext.length(), key, iv, + decryptedtext); + + /* Add a NULL terminator. We are expecting printable text */ + decryptedtext[decryptedtext_len] = '\0'; + + /* Show the decrypted text */ + printf("Decrypted text is:\n"); + printf("%s\n", decryptedtext); +} + diff --git a/system/jlib/jencrypt.hpp b/system/jlib/jencrypt.hpp index 320796a52d8..f8a504a9ad6 100644 --- a/system/jlib/jencrypt.hpp +++ b/system/jlib/jencrypt.hpp @@ -46,6 +46,7 @@ extern jlib_decl size_t aesDecrypt(const void *key, size_t keylen, const void *i extern jlib_decl void encrypt(StringBuffer &ret, const char *in); extern jlib_decl void decrypt(StringBuffer &ret, const char *in); +extern jlib_decl void xmain(); // simple inline block scrambler (shouldn't need jlib_decl) class Csimplecrypt diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 43a93fde30f..fe82e768394 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3511,6 +3511,25 @@ class JLibUnicodeTest : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_REGISTRATION( JLibUnicodeTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibUnicodeTest, "JLibUnicodeTest" ); +#include +class JLibOpensslAESTest : public CppUnit::TestFixture +{ +public: + CPPUNIT_TEST_SUITE(JLibOpensslAESTest); + CPPUNIT_TEST(test); + CPPUNIT_TEST_SUITE_END(); + +protected: + + void test() + { + xmain(); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( JLibOpensslAESTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibOpensslAESTest, "JLibOpensslAESTest" ); #endif // _USE_CPPUNIT From d557560149ba3c2692087089b363683ca707841d Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 8 Nov 2023 17:01:02 +0000 Subject: [PATCH 5/5] WIP Signed-off-by: Richard Chapman --- system/jlib/jencrypt.cpp | 195 ++++++++++++++++++++------------------- 1 file changed, 99 insertions(+), 96 deletions(-) diff --git a/system/jlib/jencrypt.cpp b/system/jlib/jencrypt.cpp index 9ba53cc4fe5..711a24b4963 100644 --- a/system/jlib/jencrypt.cpp +++ b/system/jlib/jencrypt.cpp @@ -21,6 +21,10 @@ #ifdef _USE_OPENSSL #include "ske.hpp" +#include +#include +#include +#include #endif @@ -1829,18 +1833,107 @@ size_t aesDecrypt(const void *key, size_t keylen, const void *input, size_t inle } // end of namespace jlib +#ifdef _USE_OPENSSL +MemoryBuffer &aesEncrypt_ssl(const void *key, unsigned keylen, const void *plaintext, size_t plaintext_len, MemoryBuffer &output) +{ + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + if (!ctx) + throw makeStringException(0, "Crap"); + unsigned char iv[16] = { 0 }; + switch (keylen) + { + case 32: + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, (const unsigned char *) key, iv)) + throw makeStringException(0, "Crap"); + break; + case 24: + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_192_cbc(), NULL, (const unsigned char *) key, iv)) + throw makeStringException(0, "Crap"); + break; + case 16: + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_128_cbc(), NULL, (const unsigned char *) key, iv)) + throw makeStringException(0, "Crap"); + break; + default: + throw makeStringException(0, "Crap"); + } + byte *ciphertext = (byte *) output.reserve(plaintext_len + 100); + int ciphertext_len = 0; + int thislen = 0; + if(1 != EVP_EncryptUpdate(ctx, ciphertext, &thislen, (const unsigned char *) plaintext, plaintext_len)) + throw makeStringException(0, "Crap"); + ciphertext_len += thislen; + if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + ciphertext_len, &thislen)) + throw makeStringException(0, "Crap"); + ciphertext_len += thislen; + EVP_CIPHER_CTX_free(ctx); + output.setLength(ciphertext_len); + return output; +} + +MemoryBuffer &aesDecrypt_ssl(const void *key, size_t keylen, const void *ciphertext, size_t ciphertext_len, MemoryBuffer &output) +{ + EVP_CIPHER_CTX *ctx; + + int thislen = 0; + int plaintext_len = 0; + + if(!(ctx = EVP_CIPHER_CTX_new())) + throw makeStringException(0, "Crap"); + + unsigned char iv[16] = { 0 }; + switch (keylen) + { + case 32: + if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, (const unsigned char *) key, iv)) + throw makeStringException(0, "Crap"); + break; + case 24: + if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_192_cbc(), NULL, (const unsigned char *) key, iv)) + throw makeStringException(0, "Crap"); + break; + case 16: + if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_128_cbc(), NULL, (const unsigned char *) key, iv)) + throw makeStringException(0, "Crap"); + break; + default: + throw makeStringException(0, "Crap"); + } + byte *plaintext = (byte *) output.reserve(ciphertext_len + 100); + if(1 != EVP_DecryptUpdate(ctx, plaintext, &thislen, (const unsigned char *) ciphertext, ciphertext_len)) + throw makeStringException(0, "Crap"); + plaintext_len += thislen; + + if(1 != EVP_DecryptFinal_ex(ctx, plaintext + plaintext_len, &thislen)) + throw makeStringException(0, "Crap"); + plaintext_len += thislen; + EVP_CIPHER_CTX_free(ctx); + return output; +} + +#endif + MemoryBuffer &aesEncrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output) { +#ifdef _USE_OPENSSL + return aesEncrypt_ssl(key, keylen, input, inlen, output); +#else return jlib::aesEncrypt(key, keylen, input, inlen, output); +#endif } MemoryBuffer &aesDecrypt(const void *key, size_t keylen, const void *input, size_t inlen, MemoryBuffer &output) { +#ifdef _USE_OPENSSL + return aesDecrypt_ssl(key, keylen, input, inlen, output); +#else return jlib::aesDecrypt(key, keylen, input, inlen, output); +#endif } size_t aesDecrypt(const void *key, size_t keylen, const void *input, size_t inlen, void *output, size_t outlen) { + // MORE - add openssl version! return jlib::aesDecrypt(key, keylen, input, inlen, output, outlen); } @@ -1874,85 +1967,6 @@ void decrypt(StringBuffer &ret, const char *in) } } - - -#include -#include -#include -#include - -void handleErrors(void) -{ - ERR_print_errors_fp(stderr); - abort(); - -} - -MemoryBuffer &aesEncrypt_ssl(const void *key, unsigned keylen, const void *plaintext, size_t plaintext_len, MemoryBuffer &output) -{ - assertex(keylen==32); - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - if (!ctx) - throw makeStringException(0, "Crap"); - unsigned char iv[16] = { 0 }; - if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, (const unsigned char *) key, iv)) - throw makeStringException(0, "Crap"); - byte *ciphertext = (byte *) output.reserve(plaintext_len + 100); - int ciphertext_len = 0; - if(1 != EVP_EncryptUpdate(ctx, ciphertext, &ciphertext_len, (const unsigned char *) plaintext, plaintext_len)) - throw makeStringException(0, "Crap"); - if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + ciphertext_len, &ciphertext_len)) - throw makeStringException(0, "Crap"); - EVP_CIPHER_CTX_free(ctx); - output.setLength(ciphertext_len); - return output; -} - -int aesDecrypt_ssl(unsigned char *ciphertext, int ciphertext_len, unsigned char *key, - unsigned char *iv, unsigned char *plaintext) -{ - EVP_CIPHER_CTX *ctx; - - int len; - - int plaintext_len; - - /* Create and initialise the context */ - if(!(ctx = EVP_CIPHER_CTX_new())) - handleErrors(); - - /* - * Initialise the decryption operation. IMPORTANT - ensure you use a key - * and IV size appropriate for your cipher - * In this example we are using 256 bit AES (i.e. a 256 bit key). The - * IV size for *most* modes is the same as the block size. For AES this - * is 128 bits - */ - if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, key, iv)) - handleErrors(); - - /* - * Provide the message to be decrypted, and obtain the plaintext output. - * EVP_DecryptUpdate can be called multiple times if necessary. - */ - if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) - handleErrors(); - plaintext_len = len; - - /* - * Finalise the decryption. Further plaintext bytes may be written at - * this stage. - */ - if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) - handleErrors(); - plaintext_len += len; - - /* Clean up */ - EVP_CIPHER_CTX_free(ctx); - - return plaintext_len; -} - void xmain (void) { /* @@ -1967,36 +1981,25 @@ void xmain (void) 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31 }; - /* A 128 bit IV */ - unsigned char iv[16] = { 0 }; - /* Message to be encrypted */ unsigned char *plaintext = (unsigned char *)"The quick brown fox jumps over the lazy dog"; - MemoryBuffer ciphertext; - - /* Buffer for the decrypted text */ - unsigned char decryptedtext[128]; - - int decryptedtext_len, ciphertext_len; + MemoryBuffer ciphertext, decrypted; - /* Encrypt the plaintext */ - aesEncrypt_ssl(key, 32, plaintext, strlen ((char *)plaintext), ciphertext); - ciphertext_len = ciphertext.length(); + //aesEncrypt_ssl(key, 32, plaintext, strlen ((char *)plaintext), ciphertext); + jlib::aesEncrypt(key, 32, plaintext, strlen ((char *)plaintext), ciphertext); - /* Do something useful with the ciphertext here */ printf("Ciphertext is:\n"); BIO_dump_fp (stdout, ciphertext.bytes(), ciphertext.length()); /* Decrypt the ciphertext */ - decryptedtext_len = aesDecrypt_ssl((unsigned char *) ciphertext.bytes(), ciphertext.length(), key, iv, - decryptedtext); + aesDecrypt_ssl(key, 32, ciphertext.bytes(), ciphertext.length(), decrypted); /* Add a NULL terminator. We are expecting printable text */ - decryptedtext[decryptedtext_len] = '\0'; + decrypted.append('\0'); /* Show the decrypted text */ printf("Decrypted text is:\n"); - printf("%s\n", decryptedtext); + printf("%s\n", (const char *) decrypted.bytes()); }