Commit 6d8004a2 authored by Carsten Haitzler's avatar Carsten Haitzler

add lz4 compression options to eet. remove amalgamated build too.



SVN revision: 71111
parent 726b7188
......@@ -587,6 +587,10 @@
* Force destruction of all pending file when shuting down eet.
2012-05-14 Carsten Haitzler (The Rasterman)
* Add LZ4/LZ4HC compression & decompression capabilities
2012-05-15 Cedric Bail
* Make eet_dictionary thread safe.
......@@ -72,23 +72,6 @@ want_signature="yes"
requirement_eet=""
### Additional options to configure
# Amalgamation
AC_ARG_ENABLE([amalgamation],
[AC_HELP_STRING([--enable-amalgamation], [enable generation of one single file with all source code in it, helps compiler optimizations.])],
[if test "x${enableval}" = "xyes"; then
do_amalgamation="yes"
else
do_amalgamation="no"
fi
],
[do_amalgamation="no"]
)
AM_CONDITIONAL(EET_AMALGAMATION, test "x${do_amalgamation}" = "xyes")
EFL_ENABLE_BIN([eet])
# Old eet file format support
......@@ -516,7 +499,6 @@ echo "------------------------------------------------------------------------"
echo
echo "Configuration Options Summary:"
echo
echo " Amalgamation.........: ${do_amalgamation}"
echo " Secure layer.........: ${secure_layer}"
if test "x${have_gnutls}" = "xyes" || test "x${have_openssl}" = "xyes" ; then
echo " Cipher support.....: ${have_cipher}"
......
......@@ -256,6 +256,39 @@ typedef enum _Eet_Error
EET_ERROR_DECRYPT_FAILED /**< Could not decrypt contents */
} Eet_Error; /**< Eet error identifiers */
/**
* @}
*/
/**
* @defgroup Eet_Compression Eet Compression Levels
* Compression modes/levels supported by Eet.
*
* @{
*/
/**
* @enum _Eet_Compression
* All the compression modes known by Eet.
*/
typedef enum _Eet_Compression
{
EET_COMPRESSION_NONE = 0, /**< No compression at all */
EET_COMPRESSION_DEFAULT = 1, /**< Default compression (Zlib) */
EET_COMPRESSION_LOW = 2, /**< Fast but minimal compression (Zlib) */
EET_COMPRESSION_MED = 6, /**< Medium compression level (Zlib) */
EET_COMPRESSION_HI = 9, /**< Slow but high compression level (Zlib) */
EET_COMPRESSION_VERYFAST = 10, /**< Very fast, but lower compression ratio (LZ4HC) */
EET_COMPRESSION_SUPERFAST = 11, /**< Very fast, but lower compression ratio (faster to compress than EET_COMPRESSION_VERYFAST) (LZ4) */
EET_COMPRESSION_LOW2 = 3, /**< Space filler for compatibility. Don't use it */
EET_COMPRESSION_MED1 = 4, /**< Space filler for compatibility. Don't use it */
EET_COMPRESSION_MED2 = 5, /**< Space filler for compatibility. Don't use it */
EET_COMPRESSION_HI1 = 7, /**< Space filler for compatibility. Don't use it */
EET_COMPRESSION_HI2 = 8 /**< Space filler for compatibility. Don't use it */
} Eet_Compression; /**< Eet compression modes */
/**
* @}
*/
......
......@@ -125,6 +125,8 @@ struct _Eet_File_Node
unsigned int size;
unsigned int data_size;
unsigned char compression_type;
unsigned char free_name : 1;
unsigned char compression : 1;
unsigned char ciphered : 1;
......
......@@ -4,6 +4,7 @@ MAINTAINERCLEANFILES = Makefile.in
AM_CPPFLAGS = \
-I. \
-I$(top_srcdir)/src/lib \
-I$(top_srcdir)/src/lib/lz4 \
-I$(top_builddir)/src/lib \
-DPACKAGE_BIN_DIR=\"$(bindir)\" \
-DPACKAGE_LIB_DIR=\"$(libdir)\" \
......@@ -21,7 +22,7 @@ includesdir = $(includedir)/eet-@VMAJ@
lib_LTLIBRARIES = libeet.la
base_sources = \
libeet_la_SOURCES = \
eet_alloc.c \
eet_lib.c \
eet_data.c \
......@@ -30,128 +31,11 @@ eet_cipher.c \
eet_dictionary.c \
eet_node.c \
eet_utils.c \
eet_connection.c
if EET_AMALGAMATION
nodist_libeet_la_SOURCES = eet_amalgamation.c
eet_amalgamation.c: $(base_sources) Makefile
-rm -f eet_amalgamation.c
@echo "#ifdef HAVE_CONFIG_H" >> eet_amalgamation.c
@echo "# include \"config.h\"" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#ifdef HAVE_ALLOCA_H" >> eet_amalgamation.c
@echo "# include <alloca.h>" >> eet_amalgamation.c
@echo "#elif defined __GNUC__" >> eet_amalgamation.c
@echo "# define alloca __builtin_alloca" >> eet_amalgamation.c
@echo "#elif defined _AIX" >> eet_amalgamation.c
@echo "# define alloca __alloca" >> eet_amalgamation.c
@echo "#elif defined _MSC_VER" >> eet_amalgamation.c
@echo "# include <malloc.h>" >> eet_amalgamation.c
@echo "# define alloca _alloca" >> eet_amalgamation.c
@echo "#else" >> eet_amalgamation.c
@echo "# include <stddef.h>" >> eet_amalgamation.c
@echo "# ifdef __cplusplus" >> eet_amalgamation.c
@echo "#extern \"C\"" >> eet_amalgamation.c
@echo "# endif" >> eet_amalgamation.c
@echo "#void *alloca (size_t);" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#include <stdio.h>" >> eet_amalgamation.c
@echo "#include <string.h>" >> eet_amalgamation.c
@echo "#include <math.h>" >> eet_amalgamation.c
@echo "#include <ctype.h>" >> eet_amalgamation.c
@echo "#include <limits.h>" >> eet_amalgamation.c
@echo "#include <sys/types.h>" >> eet_amalgamation.c
@echo "#include <sys/stat.h>" >> eet_amalgamation.c
@echo "#ifdef HAVE_SIGNATURE" >> eet_amalgamation.c
@echo "# include <sys/mman.h>" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#include <setjmp.h>" >> eet_amalgamation.c
@echo "#include <errno.h>" >> eet_amalgamation.c
@echo "#include <time.h>" >> eet_amalgamation.c
@echo "#include <fnmatch.h>" >> eet_amalgamation.c
@echo "#include <fcntl.h>" >> eet_amalgamation.c
@echo "#ifdef _WIN32" >> eet_amalgamation.c
@echo "# include <winsock2.h>" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#ifndef _MSC_VER" >> eet_amalgamation.c
@echo "# include <unistd.h>" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#ifdef HAVE_NETINET_IN_H" >> eet_amalgamation.c
@echo "# include <netinet/in.h>" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#ifdef HAVE_GNUTLS" >> eet_amalgamation.c
@echo "# include <gnutls/gnutls.h>" >> eet_amalgamation.c
@echo "# include <gcrypt.h>" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#ifdef HAVE_OPENSSL" >> eet_amalgamation.c
@echo "# include <openssl/err.h>" >> eet_amalgamation.c
@echo "# include <openssl/evp.h>" >> eet_amalgamation.c
@echo "# include <openssl/sha.h>" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#ifdef HAVE_SIGNATURE" >> eet_amalgamation.c
@echo "# ifdef HAVE_GNUTLS" >> eet_amalgamation.c
@echo "# include <gnutls/gnutls.h>" >> eet_amalgamation.c
@echo "# include <gnutls/x509.h>" >> eet_amalgamation.c
@echo "# else" >> eet_amalgamation.c
@echo "# include <openssl/rsa.h>" >> eet_amalgamation.c
@echo "# include <openssl/objects.h>" >> eet_amalgamation.c
@echo "# include <openssl/err.h>" >> eet_amalgamation.c
@echo "# include <openssl/ssl.h>" >> eet_amalgamation.c
@echo "# include <openssl/dh.h>" >> eet_amalgamation.c
@echo "# include <openssl/dsa.h>" >> eet_amalgamation.c
@echo "# include <openssl/evp.h>" >> eet_amalgamation.c
@echo "# include <openssl/x509.h>" >> eet_amalgamation.c
@echo "# include <openssl/pem.h>" >> eet_amalgamation.c
@echo "# endif" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#ifdef HAVE_CIPHER" >> eet_amalgamation.c
@echo "# ifdef HAVE_GNUTLS" >> eet_amalgamation.c
@echo "# include <gnutls/x509.h>" >> eet_amalgamation.c
@echo "# include <gcrypt.h>" >> eet_amalgamation.c
@echo "# else" >> eet_amalgamation.c
@echo "# include <openssl/evp.h>" >> eet_amalgamation.c
@echo "# include <openssl/hmac.h>" >> eet_amalgamation.c
@echo "# include <openssl/rand.h>" >> eet_amalgamation.c
@echo "# endif" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#include <zlib.h>" >> eet_amalgamation.c
@echo "#include <jpeglib.h>" >> eet_amalgamation.c
@echo "#ifdef HAVE_EVIL" >> eet_amalgamation.c
@echo "# include <Evil.h>" >> eet_amalgamation.c
@echo "#endif" >> eet_amalgamation.c
@echo "#include <Eet.h>" >> eet_amalgamation.c
@echo "#include \"Eet_private.h\"" >> eet_amalgamation.c
@echo "#include \"Eet.h\"" >> eet_amalgamation.c
@for f in $(base_sources); do \
if [ `echo $$f | sed -e 's/^...\(.\).*/\1/'` != '/' ]; then \
file="$(srcdir)/$$f" ; \
else \
file="$$f" ; \
fi ; \
echo "/* file: $$file */" >> eet_amalgamation.c; \
grep -v -e '^# *include \+.\(config\|\|Evil\|Eina\|stdio\|string\|math\|ctype\|limits\|sys/types\|sys/stat\|sys/mman\|setjmp\|errno\|time\|fnmatch\|fcntl\|winsock2\|unistd\|netinet/in\|gnutls/gnutls\|gcrypt\|gnutls/x509\|openssl/rsa\|openssl/objects\|openssl/err\|openssl/ssl\|openssl/dh\|openssl/dsa\|openssl/evp\|openssl/pem\|openssl/sha\|openssl/hmac\|openssl/x509\|openssl/rand\|zlib\|jpeglib\|Eet_private\|Eet\)[.]h.*' $$file >> eet_amalgamation.c; \
done
@echo "eet_amalgamation.c generated"
else
libeet_la_SOURCES = $(base_sources)
endif
eet_connection.c \
lz4/lz4.c \
lz4/lz4.h \
lz4/lz4hc.c \
lz4/lz4hc.h
libeet_la_CFLAGS = @EET_CFLAGS@ @DEBUG_CFLAGS@
libeet_la_LIBADD = @GNUTLS_LIBS@ @OPENSSL_LIBS@ @EFL_COVERAGE_LIBS@ @EET_LIBS@ @EINA_LIBS@ @EVIL_LIBS@ -lz -ljpeg -lm
......
......@@ -40,6 +40,9 @@ void *alloca(size_t);
#include "Eet.h"
#include "Eet_private.h"
#include "lz4.h"
#include "lz4hc.h"
/*---*/
typedef struct _JPEG_error_mgr *emptr;
......@@ -742,55 +745,68 @@ eet_data_image_lossless_compressed_convert(int *size,
}
{
unsigned char *d;
unsigned char *comp;
int *header;
int ret;
uLongf buflen;
d = malloc((w * h * 4) + (8 * 4));
if (!d)
return NULL;
unsigned char *d, *comp;
int *header, ret, ok = 1;
uLongf buflen = 0;
buflen = (((w * h * 101) / 100) + 3) * 4;
ret = LZ4_compressBound((w * h * 4));
if ((ret > 0) && ((uLongf)ret > buflen)) buflen = ret;
comp = malloc(buflen);
if (!comp)
if (!comp) return NULL;
switch (compression)
{
case EET_COMPRESSION_VERYFAST:
ret = LZ4_compressHC((const char *)data, (char *)comp,
(w * h * 4));
if (ret <= 0) ok = 0;
buflen = ret;
break;
case EET_COMPRESSION_SUPERFAST:
ret = LZ4_compress((const char *)data, (char *)comp,
(w * h * 4));
if (ret <= 0) ok = 0;
buflen = ret;
break;
default: /* zlib etc. */
ret = compress2((Bytef *)comp, &buflen, (Bytef *)(data),
(uLong)(w * h * 4), compression);
if (ret != Z_OK) ok = 0;
break;
}
if ((!ok) || (buflen > (w * h * 4)))
{
free(d);
free(comp);
*size = -1;
return NULL;
}
header = (int *)d;
memset(d, 0, 32);
d = malloc((8 * sizeof(int)) + buflen);
if (!d)
{
free(comp);
return NULL;
}
header = (int *)d;
memset(d, 0, 8 * sizeof(int));
header[0] = 0xac1dfeed;
header[1] = w;
header[2] = h;
header[3] = alpha;
header[4] = compression;
memcpy(d + 32, data, w * h * 4);
if (_eet_image_words_bigendian)
{
unsigned int i;
for (i = 0; i < ((w * h) + 8); i++) SWAP32(header[i]);
}
ret = compress2((Bytef *)comp, &buflen,
(Bytef *)(d + 32),
(uLong)(w * h * 4),
compression);
if (ret != Z_OK || buflen > (w * h * 4))
{
free(comp);
free(d);
*size = -1;
return NULL;
}
memcpy(d + 32, comp, buflen);
*size = (8 * 4) + buflen;
memcpy(d + (8 * sizeof(int)), comp, buflen);
*size = (8 * sizeof(int)) + buflen;
free(comp);
return d;
}
......@@ -1577,35 +1593,75 @@ _eet_data_image_decode_inside(const void *data,
w, h, row_stride);
else
{
if (src_h == h && src_w == w && row_stride == src_w * 4)
if ((src_h == h) && (src_w == w) && (row_stride == src_w * 4))
{
uLongf dlen;
dlen = w * h * 4;
uncompress((Bytef *)d, &dlen, (Bytef *)body,
(uLongf)(size - 32));
switch (comp)
{
case EET_COMPRESSION_VERYFAST:
case EET_COMPRESSION_SUPERFAST:
if (LZ4_uncompress((const char *)body,
(char *)d, w * h * 4)
!= (size - 32)) return 0;
break;
default:
{
uLongf dlen = w * h * 4;
if (uncompress((Bytef *)d, &dlen, (Bytef *)body,
(uLongf)(size - 32)) != Z_OK)
return 0;
}
break;
}
}
else
{
Bytef *dtmp;
uLongf dlen = src_w * src_h * 4;
/* FIXME: This could create a huge alloc. So compressed
data and tile could not always work. */
dtmp = malloc(dlen);
if (!dtmp)
return 0;
uncompress(dtmp, &dlen, (Bytef *)body, (uLongf)(size - 32));
_eet_data_image_copy_buffer((unsigned int *)dtmp,
src_x, src_y, src_w, d,
w, h, row_stride);
free(dtmp);
switch (comp)
{
case EET_COMPRESSION_VERYFAST:
case EET_COMPRESSION_SUPERFAST:
{
char *dtmp;
dtmp = malloc(src_w * src_h * 4);
if (!dtmp) return 0;
if (LZ4_uncompress((const char *)body,
dtmp, w * h * 4)
!= (size - 32))
{
free(dtmp);
return 0;
}
_eet_data_image_copy_buffer((unsigned int *)dtmp,
src_x, src_y, src_w, d,
w, h, row_stride);
free(dtmp);
}
break;
default:
{
Bytef *dtmp;
uLongf dlen = src_w * src_h * 4;
/* FIXME: This could create a huge alloc. So
compressed data and tile could not always work.*/
dtmp = malloc(dlen);
if (!dtmp) return 0;
if (uncompress(dtmp, &dlen, (Bytef *)body,
(uLongf)(size - 32)) != Z_OK)
{
free(dtmp);
return 0;
}
_eet_data_image_copy_buffer((unsigned int *)dtmp,
src_x, src_y, src_w, d,
w, h, row_stride);
free(dtmp);
}
}
}
}
/* Fix swapiness. */
if (_eet_image_words_bigendian)
{
......
This diff is collapsed.
This diff is collapsed.
/*
LZ4 - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2012, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
//****************************
// Simple Functions
//****************************
int LZ4_compress (const char* source, char* dest, int isize);
int LZ4_uncompress (const char* source, char* dest, int osize);
/*
LZ4_compress() :
isize : is the input size. Max supported value is ~1.9GB
return : the number of bytes written in buffer dest
or 0 if the compression fails (if LZ4_COMPRESSMIN is set)
note : destination buffer must be already allocated.
destination buffer must be sized to handle worst cases situations (input data not compressible)
worst case size evaluation is provided by function LZ4_compressBound()
LZ4_uncompress() :
osize : is the output size, therefore the original size
return : the number of bytes read in the source buffer
If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction
This function never writes beyond dest + osize, and is therefore protected against malicious data packets
note : destination buffer must be already allocated
*/
//****************************
// Advanced Functions
//****************************
int LZ4_compressBound(int isize);
/*
LZ4_compressBound() :
Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
primarily useful for memory allocation of output buffer.
isize : is the input size. Max supported value is ~1.9GB
return : maximum output size in a "worst case" scenario
note : this function is limited by "int" range (2^31-1)
*/
int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
/*
LZ4_uncompress_unknownOutputSize() :
isize : is the input size, therefore the compressed size
maxOutputSize : is the size of the destination buffer (which must be already allocated)
return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction
This function never writes beyond dest + maxOutputSize, and is therefore protected against malicious data packets
note : Destination buffer must be already allocated.
This version is slightly slower than LZ4_uncompress()
*/
int LZ4_compressCtx(void** ctx, const char* source, char* dest, int isize);
int LZ4_compress64kCtx(void** ctx, const char* source, char* dest, int isize);
/*
LZ4_compressCtx() :
This function explicitly handles the CTX memory structure.
It avoids allocating/deallocating memory between each call, improving performance when malloc is heavily invoked.
This function is only useful when memory is allocated into the heap (HASH_LOG value beyond STACK_LIMIT)
Performance difference will be noticeable only when repetitively calling the compression function over many small segments.
Note : by default, memory is allocated into the stack, therefore "malloc" is not invoked.
LZ4_compress64kCtx() :
Same as LZ4_compressCtx(), but specific to small inputs (<64KB).
isize *Must* be <64KB, otherwise the output will be corrupted.
On first call : provide a *ctx=NULL; It will be automatically allocated.
On next calls : reuse the same ctx pointer.
Use different pointers for different threads when doing multi-threading.
*/
#if defined (__cplusplus)
}
#endif
This diff is collapsed.
/*
LZ4 HC - High Compression Mode of LZ4
Header File
Copyright (C) 2011-2012, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
int LZ4_compressHC (const char* source, char* dest, int isize);
/*
LZ4_compressHC :
return : the number of bytes in compressed buffer dest
note : destination buffer must be already allocated.
To avoid any problem, size it to handle worst cases situations (input data not compressible)
Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
*/
/* Note :
Decompression functions are provided within regular LZ4 source code (see "lz4.h") (BSD license)
*/
#if defined (__cplusplus)
}
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment