Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Aes openssl #18010

Closed
wants to merge 5 commits into from
Closed

Aes openssl #18010

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
797a119
Refactor compressToBuffer to support different compression methods
richardkchapman Nov 1, 2023
8f3e079
Add AESLZ4 and AESLZ4HC modes
richardkchapman Nov 8, 2023
9bd7efd
Added hc level options for AES front end
richardkchapman Nov 8, 2023
53e916e
WIP
richardkchapman Nov 8, 2023
d557560
WIP
richardkchapman Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions common/dllserver/thorplugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -679,10 +679,7 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str
extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress)
{
mb.append((byte)0x80).append(resourceHeaderVersion);
if (compress)
compressToBuffer(mb, len, data);
else
appendToBuffer(mb, len, data);
compressToBuffer(mb, len, data, compress ? COMPRESS_METHOD_LZW : COMPRESS_METHOD_NONE);
}

extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data)
Expand Down
158 changes: 75 additions & 83 deletions system/jlib/jlzw.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -758,82 +758,61 @@ size32_t RLEExpand(void *dst,const void *src,size32_t expsize)
return (size32_t)(in-(const byte *)src);
}

void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
out.append(false);
out.append(len);
out.append(len, src);
}

void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
unsigned originalLength = out.length();
out.append(true);
out.append((size32_t)0);

if (len >= 32)
{
size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ...
Owned<ICompressor> compressor = createLZWCompressor();
void *newData = out.reserve(newSize);
compressor->open(newData, newSize);
if (compressor->write(src, len)==len)
void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method, const char *options)
{
if (method != COMPRESS_METHOD_NONE)
{
unsigned originalLength = out.length();
// For back-compatibility, we always store COMPRESS_METHOD_LZW as 1 as earlier versions stored a boolean here
// rather than an enum
// This means that compressToBuffer/decompressToBuffer cannot bs used for rowdiff compression - this is not likely to be an issue
// Alternative would be a separate enum for compressToBuffer formats, but that seems more likely to cause confusion
out.append((byte) (method == COMPRESS_METHOD_LZW ? COMPRESS_METHOD_LZWLEGACY : method));
out.append((size32_t)0);
if (len >= 32)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: Could be moved to the outer if ()

{
compressor->close();
size32_t compressedLen = compressor->buflen();
out.setWritePos(originalLength + sizeof(bool));
out.append(compressedLen);
out.setWritePos(originalLength + sizeof(bool) + sizeof(size32_t) + compressedLen);
return;
size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ...
Owned<ICompressor> compressor = queryCompressHandler(method)->getCompressor(options);
void *newData = out.reserve(newSize);
compressor->open(newData, newSize);
if (compressor->write(src, len)==len)
{
compressor->close();
size32_t compressedLen = compressor->buflen();
out.setWritePos(originalLength + sizeof(bool));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

picky: Should now be sizeof(byte)(!) and line 784

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could be sizeof(CompressMethod), if the enum was defined with type:

enum CompressionMethod : byte

.. would also remove need for some typecasts elsewhere.

out.append(compressedLen);
out.setWritePos(originalLength + sizeof(bool) + sizeof(size32_t) + compressedLen);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not new, but would be more readable, if these were in order they were appended to buffer.

return;
}
}

// all or don't compress
out.setWritePos(originalLength);
}

// all or don't compress
out.setWritePos(originalLength);
appendToBuffer(out, len, src);
}

void decompressToBuffer(MemoryBuffer & out, const void * src)
{
Owned<IExpander> expander = createLZWExpander();
unsigned outSize = expander->init(src);
void * buff = out.reserve(outSize);
expander->expand(buff);
out.append((byte) COMPRESS_METHOD_NONE);
out.append(len);
out.append(len, src);
}


void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in)
void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options)
{
bool compressed;
size32_t srcLen;
in.read(compressed).read(srcLen);
if (compressed)
decompressToBuffer(out, in.readDirect(srcLen));
else
unsigned char _method;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

byte method (or byte ComressMethod _method if enum is : byte) ?

in.read(_method).read(srcLen);
CompressionMethod method = (CompressionMethod) _method;
if (method==COMPRESS_METHOD_NONE)
out.append(srcLen, in.readDirect(srcLen));
}

void decompressToAttr(MemoryAttr & out, const void * src)
{
Owned<IExpander> expander = createLZWExpander();
unsigned outSize = expander->init(src);
void * buff = out.allocate(outSize);
expander->expand(buff);
}

void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in)
{
bool compressed;
size32_t srcLen;
in.read(compressed).read(srcLen);
if (compressed)
decompressToAttr(out, in.readDirect(srcLen));
else
out.set(srcLen, in.readDirect(srcLen));
{
if (method==COMPRESS_METHOD_LZWLEGACY)
method = COMPRESS_METHOD_LZW; // Back compatibilty
Owned<IExpander> expander = queryCompressHandler(method)->getExpander(options);
unsigned outSize = expander->init(in.readDirect(srcLen));
void * buff = out.reserve(outSize);
expander->expand(buff);
}
}



/*
Simple Diff compression format is

Expand Down Expand Up @@ -2875,6 +2854,18 @@ class CCompressHandlerArray : public IArrayOf<ICompressHandler>
}
return NULL;
}
ICompressHandler *lookup(CompressionMethod method) const
{
// MORE - should probably use an array, cache last lookup, or something...
// This is called quite a lot now
ForEachItemIn(h, *this)
{
ICompressHandler &handler = item(h);
if (method == handler.queryMethod())
return &handler;
}
return NULL;
}
} compressors;

typedef IIteratorOf<ICompressHandler> ICompressHandlerIterator;
Expand All @@ -2884,11 +2875,9 @@ ICompressHandlerIterator *getCompressHandlerIterator()
return new ArrayIIteratorOf<IArrayOf<ICompressHandler>, ICompressHandler, ICompressHandlerIterator>(compressors);
}



bool addCompressorHandler(ICompressHandler *handler)
{
if (compressors.lookup(handler->queryType()))
if (compressors.lookup(handler->queryMethod()))
{
handler->Release();
return false; // already registered
Expand All @@ -2908,38 +2897,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
{
class CCompressHandlerBase : implements ICompressHandler, public CInterface
{
StringAttr type;
public:
IMPLEMENT_IINTERFACE;
CCompressHandlerBase(const char *_type) : type(_type) { }
// ICompressHandler
virtual const char *queryType() const { return type; }
};
class CFLZCompressHandler : public CCompressHandlerBase
{
public:
CFLZCompressHandler() : CCompressHandlerBase("FLZ") { }
virtual const char *queryType() const { return "FLZ"; }
virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_FASTLZ; }
virtual ICompressor *getCompressor(const char *options) { return createFastLZCompressor(); }
virtual IExpander *getExpander(const char *options) { return createFastLZExpander(); }
};
class CLZ4CompressHandler : public CCompressHandlerBase
{
public:
CLZ4CompressHandler() : CCompressHandlerBase("LZ4") { }
virtual const char *queryType() const { return "LZ4"; }
virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4; }
virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, false); }
virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); }
};
class CLZ4HCCompressHandler : public CCompressHandlerBase
{
public:
CLZ4HCCompressHandler() : CCompressHandlerBase("LZ4HC") { }
virtual const char *queryType() const { return "LZ4HC"; }
virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4HC; }
virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, true); }
virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); }
};
class CAESCompressHandler : public CCompressHandlerBase
{
public:
CAESCompressHandler() : CCompressHandlerBase("AES") { }
virtual const char *queryType() const { return "AES"; }
virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZW); }
virtual ICompressor *getCompressor(const char *options)
{
assertex(options);
Expand All @@ -2954,34 +2943,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
class CDiffCompressHandler : public CCompressHandlerBase
{
public:
CDiffCompressHandler() : CCompressHandlerBase("DIFF") { }
virtual const char *queryType() const { return "DIFF"; }
virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; }
virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); }
virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); }
};
class CRDiffCompressHandler : public CCompressHandlerBase
{
public:
CRDiffCompressHandler() : CCompressHandlerBase("RDIFF") { }
virtual const char *queryType() const { return "RDIFF"; } // Synonym for DIFF
virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; }
virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); }
virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); }
};
class CRandRDiffCompressHandler : public CCompressHandlerBase
{
public:
CRandRDiffCompressHandler() : CCompressHandlerBase("RANDROW") { }
virtual const char *queryType() const { return "RANDROW"; }
virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_RANDROW; }
virtual ICompressor *getCompressor(const char *options) { return createRandRDiffCompressor(); }
virtual IExpander *getExpander(const char *options) { UNIMPLEMENTED; } // Expander has a different interface
};
class CLZWCompressHandler : public CCompressHandlerBase
{
public:
CLZWCompressHandler() : CCompressHandlerBase("LZW") { }
virtual const char *queryType() const { return "LZW"; }
virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZW; }
virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(true); }
virtual IExpander *getExpander(const char *options) { return createLZWExpander(true); }
};
addCompressorHandler(new CLZWCompressHandler());
addCompressorHandler(new CAESCompressHandler());
addCompressorHandler(new CDiffCompressHandler());
addCompressorHandler(new CLZWCompressHandler());
addCompressorHandler(new CRDiffCompressHandler());
addCompressorHandler(new CRandRDiffCompressHandler());
addCompressorHandler(new CFLZCompressHandler());
Expand All @@ -2999,8 +2992,7 @@ ICompressHandler *queryCompressHandler(const char *type)

ICompressHandler *queryCompressHandler(CompressionMethod method)
{
//Could be more efficient, but doesn't matter
return compressors.lookup(translateFromCompMethod(method));
return compressors.lookup(method);
}

void setDefaultCompressor(const char *type)
Expand Down
12 changes: 5 additions & 7 deletions system/jlib/jlzw.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ enum CompressionMethod


COMPRESS_METHOD_AES = 0x80,
COMPRESS_METHOD_LZWLEGACY = 1, // Matches value of boolean 'true' used to indicate LZW compression by legacy compressToBuffer
};


Expand Down Expand Up @@ -117,13 +118,9 @@ extern jlib_decl ICompressor *createRandRDiffCompressor(); // similar to RDiffCo
extern jlib_decl IRandRowExpander *createRandRDiffExpander(); // NB only supports fixed row size


//Some helper functions to make it easy to compress/decompress to memorybuffers.
extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src);
extern jlib_decl void decompressToBuffer(MemoryBuffer & out, const void * src);
extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in);
extern jlib_decl void decompressToAttr(MemoryAttr & out, const void * src);
extern jlib_decl void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in);
extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src); //format as failed compression
// Helper functions to make it easy to compress/decompress to memorybuffers.
extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method=COMPRESS_METHOD_LZW, const char *options=nullptr);
extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options=nullptr);


interface ICompressedFileIO: extends IFileIO
Expand Down Expand Up @@ -158,6 +155,7 @@ extern jlib_decl IPropertyTree *getBlockedFileDetails(IFile *file);
interface ICompressHandler : extends IInterface
{
virtual const char *queryType() const = 0;
virtual CompressionMethod queryMethod() const = 0;
virtual ICompressor *getCompressor(const char *options=NULL) = 0;
virtual IExpander *getExpander(const char *options=NULL) = 0;
};
Expand Down
Loading