Commit 51ef3035 authored by Bernhard Link's avatar Bernhard Link

some start of .xz generation support (using liblzma)

parent fc11e224
2014-05-10 Bernhard R. Link <brlink@debian.org>
* add support for linking against liblzma
and generating .xz Indices.
(no changes to decompressing code yet, that still
needs xzcat available)
2014-03-18 Bernhard R. Link <brlink@debian.org>
* update ignore source packages with ExtraSourceOnly by default,
unless the new OmitExtraSourceOnly option is set in conf/updates
......
......@@ -84,6 +84,26 @@ AC_ARG_WITH(libbz2,
AC_CHECK_LIB(bz2,BZ2_bzCompressInit,,[AC_MSG_WARN(["no libbz2 found, compiling without"])],)
])
AC_ARG_WITH(liblzma,
[ --with-liblzma=path|yes|no Give path to prefix liblzma was installed with],[dnl
case "$withval" in
no)
;;
yes)
AC_CHECK_LIB(lzma,lzma_easy_encoder,,[AC_MSG_ERROR(["no liblzma found, despite being told to use it"])],)
;;
*)
AC_CHECK_LIB(lzma,lzma_easy_encoder,[dnl
AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_LIBLZMA))
LIBS="$LIBS -L$withval/lib -llzma"
CPPFLAGS="$CPPFLAGS -I$withval/include"
],[AC_MSG_ERROR(["no liblzma found, despite being told to use it"])],[-L$withval/lib])
;;
esac
],[dnl without --with-liblzma we look for it but not finding it is no error:
AC_CHECK_LIB(lzma,lzma_easy_encoder,,[AC_MSG_WARN(["no liblzma found, compiling without"])],)
])
ARCHIVELIBS=""
ARCHIVECPP=""
AH_TEMPLATE([HAVE_LIBARCHIVE],[Defined if libarchive is available])
......
......@@ -39,7 +39,7 @@
retvalue contentsoptions_parse(struct distribution *distribution, struct configiterator *iter) {
enum contentsflags {
cf_disable, cf_dummy, cf_udebs, cf_nodebs,
cf_uncompressed, cf_gz, cf_bz2,
cf_uncompressed, cf_gz, cf_bz2, cf_xz,
cf_percomponent, cf_allcomponents,
cf_compatsymlink, cf_nocompatsymlink,
cf_COUNT
......@@ -55,6 +55,7 @@ retvalue contentsoptions_parse(struct distribution *distribution, struct configi
{"allcomponents", cf_allcomponents},
{"compatsymlink", cf_compatsymlink},
{"nocompatsymlink", cf_nocompatsymlink},
{".xz", cf_xz},
{".bz2", cf_bz2},
{".gz", cf_gz},
{".", cf_uncompressed},
......@@ -103,6 +104,16 @@ retvalue contentsoptions_parse(struct distribution *distribution, struct configi
config_filename(iter), config_line(iter));
flags[cf_bz2] = false;
}
#endif
#ifndef HAVE_LIBLZMA
if (flags[cf_xz]) {
fprintf(stderr,
"Warning: Ignoring request to generate .xz'ed Contents files.\n"
"(xz support disabled at build time.)\n"
"Request was in %s in the Contents header ending in line %u\n",
config_filename(iter), config_line(iter));
flags[cf_xz] = false;
}
#endif
distribution->contents.compressions = 0;
if (flags[cf_uncompressed])
......@@ -112,6 +123,10 @@ retvalue contentsoptions_parse(struct distribution *distribution, struct configi
#ifdef HAVE_LIBBZ2
if (flags[cf_bz2])
distribution->contents.compressions |= IC_FLAG(ic_bzip2);
#endif
#ifdef HAVE_LIBLZMA
if (flags[cf_xz])
distribution->contents.compressions |= IC_FLAG(ic_xz);
#endif
distribution->contents.flags.udebs = flags[cf_udebs];
distribution->contents.flags.nodebs = flags[cf_nodebs];
......
......@@ -1387,12 +1387,12 @@ Choose what kind of Index files to export. The first
part describes what the Index file shall be called.
The second argument determines the name of a Release
file to generate or not to generate if missing.
Then at least one of "\fB.\fP", "\fB.gz\fP" or "\fB.bz2\fP"
Then at least one of "\fB.\fP", "\fB.gz\fP", "\fB.xz\fP" or "\fB.bz2\fP"
specifying whether to generate uncompressed output, gzipped
output, bzip2ed output or any combination.
(bzip2 is only available when compiled with bzip2 support,
so it might not be available when you compiled it on your
own).
own, same for xz and liblzma).
If an argument not starting with dot follows,
it will be executed after all index files are generated.
(See the examples for what argument this gets).
......@@ -1419,7 +1419,8 @@ If there is a \fBudebs\fP keyword, \fB.udeb\fPs are also listed
(in a file called \fBuContents\-\fP\fIarchitecture\fP.)
If there is a \fBnodebs\fP keyword, \fB.deb\fPs are not listed.
(Only useful together with \fBudebs\fP)
If there is at least one of the keywords \fB.\fP, \fB.gz\fP and/or \fB.bz2\fP,
If there is at least one of the keywords
\fB.\fP, \fB.gz\fP, \fB\.xz\fP and/or \fB.bz2\fP,
the Contents files are written uncompressed, gzipped and/or bzip2ed instead
of only gzipped.
......
......@@ -208,6 +208,10 @@ retvalue exportmode_set(struct exportmode *mode, struct configiterator *iter) {
else if (word[1] == 'b' && word[2] == 'z' && word[3] == '2' &&
word[4] == '\0')
mode->compressions |= IC_FLAG(ic_bzip2);
#endif
#ifdef HAVE_LIBLZMA
else if (word[1] == 'x' && word[2] == 'z' &&word[3] == '\0')
mode->compressions |= IC_FLAG(ic_xz);
#endif
else {
fprintf(stderr,
......
......@@ -32,6 +32,9 @@
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif
#ifdef HAVE_LIBBZ2
#include <lzma.h>
#endif
#define CHECKSUMS_CONTEXT visible
#include "error.h"
#include "ignore.h"
......@@ -50,6 +53,8 @@
#define INPUT_BUFFER_SIZE 1024
#define GZBUFSIZE 40960
#define BZBUFSIZE 40960
// TODO: what is the correct value here:
#define XZBUFSIZE 40960
struct release {
/* The base-directory of the distribution we are exporting */
......@@ -363,6 +368,10 @@ static char *calc_compressedname(const char *name, enum indexcompression ic) {
#ifdef HAVE_LIBBZ2
case ic_bzip2:
return calc_addsuffix(name, "bz2");
#endif
#ifdef HAVE_LIBLZMA
case ic_xz:
return calc_addsuffix(name, "xz");
#endif
default:
assert ("Huh?" == NULL);
......@@ -517,6 +526,11 @@ struct filetorelease {
char *bzoutputbuffer; size_t bz_waiting_bytes;
bz_stream bzstream;
#endif
#ifdef HAVE_LIBLZMA
/* output buffer for bzip2 compression */
unsigned char *xzoutputbuffer; size_t xz_waiting_bytes;
lzma_stream xzstream;
#endif
};
void release_abortfile(struct filetorelease *file) {
......@@ -544,6 +558,12 @@ void release_abortfile(struct filetorelease *file) {
(void)BZ2_bzCompressEnd(&file->bzstream);
}
#endif
#ifdef HAVE_LIBLZMA
if (file->xzoutputbuffer != NULL) {
free(file->xzoutputbuffer);
lzma_end(&file->xzstream);
}
#endif
}
bool release_oldexists(struct filetorelease *file) {
......@@ -687,10 +707,35 @@ static retvalue initbzcompression(struct filetorelease *f) {
}
#endif
#ifdef HAVE_LIBLZMA
static retvalue initxzcompression(struct filetorelease *f) {
lzma_ret lret;
f->xzoutputbuffer = malloc(XZBUFSIZE);
if (FAILEDTOALLOC(f->xzoutputbuffer))
return RET_ERROR_OOM;
memset(&f->xzstream, 0, sizeof(f->xzstream));
lret = lzma_easy_encoder(&f->xzstream, 9, LZMA_CHECK_CRC64);
if (lret == LZMA_MEM_ERROR)
return RET_ERROR_OOM;
if (lret != LZMA_OK) {
fprintf(stderr, "Error from liblzma's lzma_easy_encoder: "
"%d\n", lret);
return RET_ERROR;
}
return RET_OK;
}
#endif
static const char * const ics[ic_count] = { "", ".gz"
#ifdef HAVE_LIBBZ2
, ".bz2"
#endif
#ifdef HAVE_LIBLZMA
, ".xz"
#endif
};
static inline retvalue setfilename(struct filetorelease *n, const char *relfilename, /*@null@*/const char *symlinkas, enum indexcompression ic) {
......@@ -806,6 +851,24 @@ static retvalue startfile(struct release *release, const char *filename, /*@null
return r;
}
}
#endif
#ifdef HAVE_LIBLZMA
if ((compressions & IC_FLAG(ic_xz)) != 0) {
retvalue r;
r = setfilename(n, filename, symlinkas, ic_xz);
if (!RET_WAS_ERROR(r))
r = openfile(release->dirofdist, &n->f[ic_xz]);
if (RET_WAS_ERROR(r)) {
release_abortfile(n);
return r;
}
checksumscontext_init(&n->f[ic_xz].context);
r = initxzcompression(n);
if (RET_WAS_ERROR(r)) {
release_abortfile(n);
return r;
}
}
#endif
checksumscontext_init(&n->f[ic_uncompressed].context);
*file = n;
......@@ -1116,6 +1179,91 @@ static retvalue finishbz(struct filetorelease *f) {
}
#endif
#ifdef HAVE_LIBLZMA
static retvalue writexz(struct filetorelease *f) {
lzma_ret xzret;
assert (f->f[ic_xz].fd >= 0);
f->xzstream.next_in = f->buffer;
f->xzstream.avail_in = INPUT_BUFFER_SIZE;
do {
f->xzstream.next_out = f->xzoutputbuffer + f->xz_waiting_bytes;
f->xzstream.avail_out = XZBUFSIZE - f->xz_waiting_bytes;
xzret = lzma_code(&f->xzstream, LZMA_RUN);
f->xz_waiting_bytes = XZBUFSIZE - f->xzstream.avail_out;
if (xzret == LZMA_OK &&
f->xz_waiting_bytes >= XZBUFSIZE / 2) {
retvalue r;
assert (f->xz_waiting_bytes > 0);
r = writetofile(&f->f[ic_xz],
(const unsigned char *)f->xzoutputbuffer,
f->xz_waiting_bytes);
assert (r != RET_NOTHING);
if (RET_WAS_ERROR(r))
return r;
f->xz_waiting_bytes = 0;
}
} while (xzret == LZMA_OK && f->xzstream.avail_in != 0);
f->xzstream.next_in = NULL;
f->xzstream.avail_in = 0;
if (xzret != LZMA_OK) {
fprintf(stderr, "Error from liblzma's lzma_code: "
"%d\n", xzret);
return RET_ERROR;
}
return RET_OK;
}
static retvalue finishxz(struct filetorelease *f) {
lzma_ret xzret;
assert (f->f[ic_xz].fd >= 0);
f->xzstream.next_in = f->buffer;
f->xzstream.avail_in = f->waiting_bytes;
do {
f->xzstream.next_out = f->xzoutputbuffer + f->xz_waiting_bytes;
f->xzstream.avail_out = XZBUFSIZE - f->xz_waiting_bytes;
xzret = lzma_code(&f->xzstream, LZMA_FINISH);
f->xz_waiting_bytes = XZBUFSIZE - f->xzstream.avail_out;
if ((xzret == LZMA_OK || xzret == LZMA_STREAM_END)
&& f->xz_waiting_bytes > 0) {
retvalue r;
r = writetofile(&f->f[ic_xz],
(const unsigned char*)f->xzoutputbuffer,
f->xz_waiting_bytes);
assert (r != RET_NOTHING);
if (RET_WAS_ERROR(r))
return r;
f->xz_waiting_bytes = 0;
}
} while (xzret == LZMA_OK);
if (xzret != LZMA_STREAM_END) {
fprintf(stderr, "Error from liblzma's lzma_code: "
"%d\n", xzret);
return RET_ERROR;
}
assert (f->xz_waiting_bytes == 0);
lzma_end(&f->xzstream);
free(f->xzoutputbuffer);
f->xzoutputbuffer = NULL;
return RET_OK;
}
#endif
retvalue release_finishfile(struct release *release, struct filetorelease *file) {
retvalue result, r;
enum indexcompression i;
......@@ -1170,6 +1318,22 @@ retvalue release_finishfile(struct release *release, struct filetorelease *file)
}
file->f[ic_bzip2].fd = -1;
}
#endif
#ifdef HAVE_LIBLZMA
if (file->f[ic_xz].fd >= 0) {
r = finishxz(file);
if (RET_WAS_ERROR(r)) {
release_abortfile(file);
return r;
}
if (close(file->f[ic_xz].fd) != 0) {
int e = errno;
file->f[ic_xz].fd = -1;
release_abortfile(file);
return RET_ERRNO(e);
}
file->f[ic_xz].fd = -1;
}
#endif
release->new = true;
result = RET_OK;
......@@ -1187,6 +1351,7 @@ retvalue release_finishfile(struct release *release, struct filetorelease *file)
#ifdef HAVE_LIBBZ2
free(file->bzoutputbuffer);
#endif
assert(file->xzoutputbuffer == NULL);
free(file);
return result;
}
......@@ -1214,6 +1379,13 @@ static retvalue release_processbuffer(struct filetorelease *file) {
RET_UPDATE(result, r);
}
RET_UPDATE(file->state, result);
#endif
#ifdef HAVE_LIBLZMA
if (file->f[ic_xz].relativefilename != NULL) {
r = writexz(file);
RET_UPDATE(result, r);
}
RET_UPDATE(file->state, result);
#endif
return result;
}
......
......@@ -18,6 +18,9 @@ struct release;
enum indexcompression {ic_uncompressed=0, ic_gzip,
#ifdef HAVE_LIBBZ2
ic_bzip2,
#endif
#ifdef HAVE_LIBLZMA
ic_xz,
#endif
ic_count /* fake item to get count */
};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment