Commit 13f32907 authored by Bernhard Link's avatar Bernhard Link

Support downloading of remote indices as .bz2

parent ab6ebd7d
......@@ -17,6 +17,7 @@
way files are named there is less strange...
- many other changes are possible now and will hopefully
be implemented soon.
- support downloading .bz2 indices
2008-08-22
* some internal cleanup preparing for future changes...
......
......@@ -6,7 +6,6 @@ TODO:
switch to only include if source is present
action to redownload missing pool/ files from some update-rules (looking for md5sum)
check for harmful override entries (Package, Source, ....)
also support .bz2 Index file download
also support pdiff download when mirroring
Fields to exclude architectures and components in update rules,
(or alternatively allow ! in inclusion lists).
......
......@@ -777,25 +777,25 @@ stderr
=WARNING: Single-Instance not yet supported!
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/Release'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/Release'
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/ugly/source/Sources.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/ugly/source/Sources.gz'
-v1*=Uncompress './lists/Test2toTest1_test2_ugly_Sources.gz' into './lists/Test2toTest1_test2_ugly_Sources'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/ugly/binary-${FAKEARCHITECTURE}/Packages.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/ugly/binary-${FAKEARCHITECTURE}/Packages.gz'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_ugly_${FAKEARCHITECTURE}_Packages.gz' into './lists/Test2toTest1_test2_deb_ugly_${FAKEARCHITECTURE}_Packages'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/ugly/binary-coal/Packages.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/ugly/binary-coal/Packages.gz'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_ugly_coal_Packages.gz' into './lists/Test2toTest1_test2_deb_ugly_coal_Packages'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz':
*=md5 expected: ffffffffffffffffffffffffffffffff, got: $(md5 dists/test2/stupid/source/Sources.gz)
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/binary-${FAKEARCHITECTURE}/Packages.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/binary-${FAKEARCHITECTURE}/Packages.gz'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_stupid_${FAKEARCHITECTURE}_Packages.gz' into './lists/Test2toTest1_test2_deb_stupid_${FAKEARCHITECTURE}_Packages'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/binary-coal/Packages.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/binary-coal/Packages.gz'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_stupid_coal_Packages.gz' into './lists/Test2toTest1_test2_deb_stupid_coal_Packages'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/ugly/source/Sources.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/ugly/source/Sources.bz2'
-v1*=Uncompress './lists/Test2toTest1_test2_ugly_Sources.bz2' into './lists/Test2toTest1_test2_ugly_Sources'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/ugly/binary-${FAKEARCHITECTURE}/Packages.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/ugly/binary-${FAKEARCHITECTURE}/Packages.bz2'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_ugly_${FAKEARCHITECTURE}_Packages.bz2' into './lists/Test2toTest1_test2_deb_ugly_${FAKEARCHITECTURE}_Packages'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/ugly/binary-coal/Packages.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/ugly/binary-coal/Packages.bz2'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_ugly_coal_Packages.bz2' into './lists/Test2toTest1_test2_deb_ugly_coal_Packages'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2':
*=md5 expected: ffffffffffffffffffffffffffffffff, got: $(md5 dists/test2/stupid/source/Sources.bz2)
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/binary-${FAKEARCHITECTURE}/Packages.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/binary-${FAKEARCHITECTURE}/Packages.bz2'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_stupid_${FAKEARCHITECTURE}_Packages.bz2' into './lists/Test2toTest1_test2_deb_stupid_${FAKEARCHITECTURE}_Packages'...
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/binary-coal/Packages.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/binary-coal/Packages.bz2'
-v1*=Uncompress './lists/Test2toTest1_test2_deb_stupid_coal_Packages.bz2' into './lists/Test2toTest1_test2_deb_stupid_coal_Packages'...
-v0*=There have been errors!
stdout
-v2*=Created directory "./lists"
......@@ -815,10 +815,10 @@ stderr
=WARNING: Single-Instance not yet supported!
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/Release'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/Release'
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz':
*=sha1 expected: 1111111111111111111111111111111111111111, got: $(sha1 dists/test2/stupid/source/Sources.gz)
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2':
*=sha1 expected: 1111111111111111111111111111111111111111, got: $(sha1 dists/test2/stupid/source/Sources.bz2)
-v0*=There have been errors!
stdout
EOF
......@@ -837,10 +837,10 @@ stderr
=WARNING: Single-Instance not yet supported!
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/Release'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/Release'
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz':
*=sha256 expected: 9999999999999999999999999999999999999999999999999999999999999999, got: $(sha256 dists/test2/stupid/source/Sources.gz)
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2':
*=sha256 expected: 9999999999999999999999999999999999999999999999999999999999999999, got: $(sha256 dists/test2/stupid/source/Sources.bz2)
-v0*=There have been errors!
stdout
EOF
......@@ -872,10 +872,10 @@ stderr
=WARNING: Single-Instance not yet supported!
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/Release'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/Release'
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.gz':
*=sha256 expected: 9999999999999999999999999999999999999999999999999999999999999999, got: $(sha256 dists/test2/stupid/source/Sources.gz)
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2'
*=Wrong checksum during receive of 'copy:$WORKDIR/dists/test2/stupid/source/Sources.bz2':
*=sha256 expected: 9999999999999999999999999999999999999999999999999999999999999999, got: $(sha256 dists/test2/stupid/source/Sources.bz2)
-v0*=There have been errors!
stdout
EOF
......@@ -891,7 +891,7 @@ stderr
=WARNING: Single-Instance not yet supported!
-v6*=aptmethod start 'copy:$WORKDIR/dists/test2/Release'
-v1*=aptmethod got 'copy:$WORKDIR/dists/test2/Release'
-v1*=Uncompress './lists/Test2toTest1_test2_stupid_Sources.gz' into './lists/Test2toTest1_test2_stupid_Sources'...
-v1*=Uncompress './lists/Test2toTest1_test2_stupid_Sources.bz2' into './lists/Test2toTest1_test2_stupid_Sources'...
-v6*=Called /bin/cp './lists/Test2toTest1_test2_ugly_Sources' './lists/_test1_dsc_ugly_source_Test2toTest1_Test2toTest1_test2_ugly_Sources'
-v6*=Listhook successfully returned!
-v6*=Called /bin/cp './lists/Test2toTest1_test2_deb_ugly_${FAKEARCHITECTURE}_Packages' './lists/_test1_deb_ugly_${FAKEARCHITECTURE}_Test2toTest1_Test2toTest1_test2_deb_ugly_${FAKEARCHITECTURE}_Packages'
......
......@@ -23,6 +23,9 @@
#include <assert.h>
#include <string.h>
#include <zlib.h>
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif
#include <fcntl.h>
#include "globals.h"
......@@ -128,6 +131,88 @@ static inline retvalue builtin_gunzip(const char *compressed, const char *destin
return RET_OK;
}
#ifdef HAVE_LIBBZ2
static inline retvalue builtin_bunzip2(const char *compressed, const char *destination) {
BZFILE *f;
char buffer[4096];
int bytes_read, bytes_written, written;
int destfd;
int e, i, zerror;
f = BZ2_bzopen(compressed, "r");
if( f == NULL ) {
// TODO: better error message...
fprintf(stderr, "Could not read %s\n",
compressed);
return RET_ERROR;
}
destfd = open(destination, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY, 0666);
if( destfd < 0 ) {
e = errno;
fprintf(stderr, "Error %d creating '%s': %s\n",
e, destination, strerror(e));
BZ2_bzclose(f);
return RET_ERRNO(e);
}
do {
bytes_read = BZ2_bzread(f, buffer, 4096);
if( bytes_read <= 0 )
break;
bytes_written = 0;
while( bytes_written < bytes_read ) {
written = write(destfd, buffer + bytes_written,
bytes_read - bytes_written);
if( written < 0 ) {
e = errno;
fprintf(stderr,
"Error %d writing to '%s': %s\n", e, destination, strerror(e) );
close(destfd);
BZ2_bzclose(f);
return RET_ERRNO(e);
}
bytes_written += written;
}
} while( true );
e = errno;
if( bytes_read < 0 ) {
const char *msg = BZ2_bzerror(f, &zerror);
if( zerror != Z_ERRNO ) {
fprintf(stderr, "libbz2 Error %d reading from '%s': %s!\n",
zerror, compressed, msg);
(void)close(destfd);
return RET_ERROR;
}
e = errno;
(void)BZ2_bzclose(f);
} else {
BZ2_bzclose(f);
e = errno;
zerror = 0;
}
if( zerror == Z_ERRNO ) {
fprintf(stderr, "Error %d reading from '%s': %s!\n",
e, compressed, strerror(e));
(void)close(destfd);
return RET_ERRNO(e);
} else if( zerror < 0 ) {
fprintf(stderr, "libbz2 Error %d reading from '%s'!\n",
zerror, compressed);
(void)close(destfd);
return RET_ERROR;
}
i = close(destfd);
if( i != 0 ) {
e = errno;
fprintf(stderr, "Error %d writing to '%s': %s!\n",
e, destination, strerror(e));
return RET_ERROR;
}
return RET_OK;
}
#endif
retvalue uncompress_queue_file(const char *compressed, const char *destination, enum compression compression, finishaction *action, void *privdata, bool *done_p) {
retvalue r;
......@@ -136,10 +221,20 @@ retvalue uncompress_queue_file(const char *compressed, const char *destination,
compressed, destination);
}
assert( compression == c_gzip );
(void)unlink(destination);
r = builtin_gunzip(compressed, destination);
switch( compression ) {
case c_gzip:
r = builtin_gunzip(compressed, destination);
break;
#ifdef HAVE_LIBBZ2
case c_bzip2:
r = builtin_bunzip2(compressed, destination);
break;
#endif
default:
assert( compression != compression );
}
if( RET_WAS_ERROR(r) ) {
(void)unlink(destination);
return r;
......@@ -156,10 +251,19 @@ retvalue uncompress_file(const char *compressed, const char *destination, enum c
compressed, destination);
}
assert( compression == c_gzip );
(void)unlink(destination);
r = builtin_gunzip(compressed, destination);
switch( compression ) {
case c_gzip:
r = builtin_gunzip(compressed, destination);
break;
#ifdef HAVE_LIBBZ2
case c_bzip2:
r = builtin_bunzip2(compressed, destination);
break;
#endif
default:
assert( compression != compression );
}
if( RET_WAS_ERROR(r) ) {
(void)unlink(destination);
return r;
......
......@@ -8,7 +8,11 @@ extern const char * const uncompression_suffix[c_COUNT];
* or uncompress (possibly multiple files) on the filesystem,
* controled by aptmethods */
#ifdef HAVE_LIBBZ2
#define uncompression_supported(c) ((c) == c_gzip || (c) == c_none || (c) == c_bzip2)
#else
#define uncompression_supported(c) ((c) == c_gzip || (c) == c_none)
#endif
/* we got an pid, check if it is a uncompressor we care for */
retvalue uncompress_checkpid(pid_t, int);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment