Commit d221fbba authored by Bernhard Link's avatar Bernhard Link

also support external uncompression programs for .orig.tar/.debian.tar/.tar uncompression

parent 3b2bfea5
2008-09-09
* also support external uncompression programs for
.orig.tar/.debian.tar/.tar uncompression, i.e.:
- support Section/Priority extraction from lzma compressed dsc packages
- libarchive no longer needs to be linked against zlib/libbz2
* fix some corner cases in .diff parsing
2008-09-07
* add support for external uncompression programs
- speeding up updating, as downloading and uncompressing
......
......@@ -70,7 +70,7 @@ retvalue indexfile_open(struct indexfile **file_p, const char *filename, enum co
/* +1 for *d = '\0' in eof case */
f->buffer = malloc(f->size + 1);
if( FAILEDTOALLOC(f->buffer) ) {
(void)uncompress_close(f->f);
uncompress_abort(f->f);
free(f->filename);
free(f);
return RET_ERROR_OOM;
......
......@@ -62,6 +62,7 @@
#include "log.h"
#include "copypackages.h"
#include "uncompression.h"
#include "sourceextraction.h"
#ifndef STD_BASE_DIR
#define STD_BASE_DIR "."
......@@ -387,6 +388,70 @@ ACTION_N(n, n, extractfilelist) {
return result;
}
ACTION_N(n, n, extractsourcesection) {
struct dsc_headers dsc;
struct sourceextraction *extraction;
char *section = NULL, *priority = NULL, *directory, *filename;
retvalue result, r;
bool broken;
int i;
assert( argc == 2 );
r = sources_readdsc(&dsc, argv[1], argv[1], &broken);
if( !RET_IS_OK(r) )
return r;
if( broken && !IGNORING_(brokensignatures,
"'%s' contains only broken signatures.\n"
"This most likely means the file was damaged or edited improperly\n",
argv[1]) )
return RET_ERROR;
r = dirs_getdirectory(argv[1], &directory);
if( RET_WAS_ERROR(r) ) {
sources_done(&dsc);
return r;
}
assert( RET_IS_OK(r) );
extraction = sourceextraction_init(&section, &priority);
if( FAILEDTOALLOC(extraction) ) {
sources_done(&dsc);
return RET_ERROR_OOM;
}
for( i = 0 ; i < dsc.files.names.count ; i ++ )
sourceextraction_setpart(extraction, i,
dsc.files.names.values[i]);
result = RET_OK;
while( sourceextraction_needs(extraction, &i) ) {
filename = calc_dirconcat(directory, dsc.files.names.values[i]);
if( FAILEDTOALLOC(filename) ) {
result = RET_ERROR_OOM;
break;
}
r = sourceextraction_analyse(extraction, filename);
free(filename);
if( RET_WAS_ERROR(r) ) {
result = r;
break;
}
}
free(directory);
if( RET_WAS_ERROR(result) ) {
sourceextraction_abort(extraction);
} else {
r = sourceextraction_finish(extraction);
RET_UPDATE(result, r);
}
if( RET_IS_OK(result) ) {
if( section != NULL )
printf("Section: %s\n", section);
if( priority != NULL )
printf("Priority: %s\n", priority);
}
sources_done(&dsc);
return result;
}
ACTION_F(n, n, n, y, fakeemptyfilelist) {
assert( argc == 2 );
return fakefilelist(database, argv[1]);
......@@ -1154,6 +1219,7 @@ ACTION_B(n, n, y, checkupdate) {
ACTION_C(n, n, cleanlists) {
#warning TODO: do not forget to implement this...
// TODO: when this is implemented, also log the database?
fprintf(stderr, "Sorry, not yet implemented.\n");
......@@ -2424,6 +2490,8 @@ static const struct action {
0, 0, "__dumpuncompressors"},
{"__uncompress", A_N(uncompress),
3, 3, "__uncompress .gz|.bz2|.lzma <compressed-filename> <into-filename>"},
{"__extractsourcesection", A_N(extractsourcesection),
1, 1, "__extractsourcesection <.dsc-file>"},
{"__extractcontrol", A_N(extractcontrol),
1, 1, "__extractcontrol <.deb-file>"},
{"__extractfilelist", A_N(extractfilelist),
......
......@@ -113,9 +113,7 @@ bool sourceextraction_needs(struct sourceextraction *e, int *ofs_p) {
return true;
} else if( e->debiantarfile >= 0 ) {
#ifdef HAVE_LIBARCHIVE
#warning TODO: think about adding support for other decompressions here, too.
if( e->debiancompression > c_bzip2 )
// TODO: errormessage
if( !uncompression_supported(e->debiancompression) )
return false;
*ofs_p = e->debiantarfile;
return true;
......@@ -124,9 +122,7 @@ bool sourceextraction_needs(struct sourceextraction *e, int *ofs_p) {
#endif
} else if( e->tarfile >= 0 ) {
#ifdef HAVE_LIBARCHIVE
// TODO: dito
if( e->tarcompression > c_bzip2 )
// TODO: errormessage
if( !uncompression_supported(e->tarcompression) )
return false;
*ofs_p = e->tarfile;
return true;
......@@ -139,7 +135,7 @@ bool sourceextraction_needs(struct sourceextraction *e, int *ofs_p) {
static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p, /*@null@*/char **priority_p, bool *found_p) {
size_t destlength, lines_in, lines_out;
const char *p, *s;
const char *p, *s; char *garbage;
#define BUFSIZE 4096
char buffer[BUFSIZE];
int bytes_read, used = 0, filled = 0;
......@@ -224,6 +220,15 @@ static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p,
*found_p = false;
return RET_OK;
}
if( memcmp(p, "diff ", 4) == 0 ) {
/* one exception is allowing diff lines,
* as diff -ru adds them ... */
if( !u_getline() ) {
/* strange file */
*found_p = false;
return RET_OK;
}
}
if( unlikely(memcmp(p, "--- ", 4) != 0) )
return RET_NOTHING;
if( !u_getline() )
......@@ -235,6 +240,12 @@ static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p,
s = strchr(p, '/');
if( unlikely(s == NULL) )
return RET_NOTHING;
s++;
/* another exception to allow diff output directly:
* +++ lines might have garbage after a tab... */
garbage = strchr(s, '\t');
if( garbage != NULL )
*garbage = '\0';
destlength = s - p;
/* ignore all files that are not x/debian/control */
while( strcmp(s, "debian/control") != 0 ) {
......@@ -246,16 +257,20 @@ static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p,
if( unlikely(interrupted()) )
return RET_ERROR_INTERRUPTED;
p += 4;
while( *p != ',' ) {
while( *p != ',' && *p != ' ' ) {
if( unlikely(*p == '\0') )
return RET_NOTHING;
p++;
}
p++;
lines_in = 0;
while( *p >= '0' && *p <= '9' ) {
lines_in = 10*lines_in + (*p-'0');
if( *p == ' ' )
lines_in = 1;
else {
p++;
lines_in = 0;
while( *p >= '0' && *p <= '9' ) {
lines_in = 10*lines_in + (*p-'0');
p++;
}
}
while( *p == ' ' )
p++;
......@@ -270,8 +285,10 @@ static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p,
lines_out = 10*lines_out + (*p-'0');
p++;
}
} else
} else if( *p == ' ' )
lines_out = 1;
else
return RET_NOTHING;
while( *p == ' ' )
p++;
if( unlikely(*p != '@') )
......@@ -313,8 +330,19 @@ static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p,
return RET_NOTHING;
if( unlikely(memcmp(p, "+++ ", 4) != 0) )
return RET_NOTHING;
s = p + 4 + destlength;
if( unlikely(*s != '/') )
p += 4;
s = strchr(p, '/');
if( unlikely(s == NULL) )
return RET_NOTHING;
/* another exception to allow diff output directly:
* +++ lines might have garbage after a tab... */
garbage = strchr(s, '\t');
if( garbage != NULL )
*garbage = '\0';
/* if it does not always have the same directory, then
* we cannot be sure it has no debian/control, so we
* have to fail... */
if( s != p + destlength )
return RET_NOTHING;
s++;
}
......@@ -325,20 +353,23 @@ static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p,
return RET_NOTHING;
p += 4;
p++;
while( *p != ',' ) {
while( *p != ',' && *p != ' ' ) {
if( unlikely(*p == '\0') )
return RET_NOTHING;
p++;
}
p++;
while( *p >= '0' && *p <= '9' )
if( *p == ',' ) {
p++;
while( *p >= '0' && *p <= '9' )
p++;
}
while( *p == ' ' )
p++;
if( unlikely(*(p++) != '+') )
return RET_NOTHING;
if( *(p++) != '1' || *(p++) != ',' ) {
/* a diff not starting at the first line is not yet supported */
/* a diff not starting at the first line (or not being
* more than one line) is not yet supported */
return RET_NOTHING;
}
lines_out = 0;
......@@ -464,11 +495,29 @@ static retvalue read_source_control_file(struct sourceextraction *e, struct arch
return RET_OK;
}
static int compressedfile_open(UNUSED(struct archive *a), UNUSED(void *v)) {
return ARCHIVE_OK;
}
static int compressedfile_close(UNUSED(struct archive *a), UNUSED(void *v)) {
return ARCHIVE_OK;
}
static ssize_t compressedfile_read(UNUSED(struct archive *a), void *d, const void **buffer_p) {
struct compressedfile *f = d;
// TODO malloc buffer instead
static char mybuffer[4096];
*buffer_p = mybuffer;
return uncompress_read(f, mybuffer, 4096);
}
static retvalue parse_tarfile(struct sourceextraction *e, const char *filename, enum compression c, /*@out@*/bool *found_p) {
struct archive *tar;
struct archive_entry *entry;
struct compressedfile *file;
int a;
retvalue r;
retvalue r, r2;
/* While an .tar, especially an .orig.tar can be very ugly (they should be
* pristine upstream tars, so dpkg-source works around a lot of ugliness),
......@@ -479,16 +528,20 @@ static retvalue parse_tarfile(struct sourceextraction *e, const char *filename,
return RET_NOTHING;
tar = archive_read_new();
if( FAILEDTOALLOC(tar) )
if( FAILEDTOALLOC(tar) ) {
return RET_ERROR_OOM;
}
archive_read_support_format_tar(tar);
archive_read_support_format_gnutar(tar);
if( c == c_gzip )
archive_read_support_compression_gzip(tar);
if( c == c_bzip2 )
archive_read_support_compression_bzip2(tar);
a = archive_read_open_file(tar, filename, 4096);
r = uncompress_open(&file, filename, c);
if( !RET_IS_OK(r) ) {
archive_read_finish(tar);
return r;
}
a = archive_read_open(tar, file, compressedfile_open,
compressedfile_read, compressedfile_close);
if( a != ARCHIVE_OK ) {
fprintf(stderr,
"Error %d trying to extract control information from %s:\n"
......@@ -511,35 +564,51 @@ static retvalue parse_tarfile(struct sourceextraction *e, const char *filename,
iscontrol = strcmp(s+1, "debian/control") == 0 ||
strcmp(name, "debian/control") == 0;
if( !iscontrol ) {
a = archive_read_data_skip(tar);
if( a != ARCHIVE_OK ) {
int e = archive_errno(tar);
printf("Error %d skipping %s within %s: %s\n",
e, name, filename,
archive_error_string(tar));
archive_read_finish(tar);
return (e!=0)?(RET_ERRNO(e)):RET_ERROR;
}
if( interrupted() )
return RET_ERROR_INTERRUPTED;
} else {
if( iscontrol ) {
r = read_source_control_file(e, tar, entry);
archive_read_finish(tar);
r2 = uncompress_error(file);
RET_UPDATE(r, r2);
uncompress_abort(file);
*found_p = true;
return r;
}
a = archive_read_data_skip(tar);
if( a != ARCHIVE_OK ) {
int err = archive_errno(tar);
printf("Error %d skipping %s within %s: %s\n",
err, name, filename,
archive_error_string(tar));
archive_read_finish(tar);
if( err == 0 || err == -EINVAL )
r = RET_ERROR;
else
r = RET_ERRNO(err);
r2 = uncompress_error(file);
RET_UPDATE(r, r2);
uncompress_abort(file);
return r;
}
if( interrupted() )
return RET_ERROR_INTERRUPTED;
}
if( a != ARCHIVE_EOF ) {
int e = archive_errno(tar);
int err = archive_errno(tar);
fprintf(stderr, "Error %d reading %s: %s\n",
e, filename, archive_error_string(tar));
err, filename, archive_error_string(tar));
archive_read_finish(tar);
return (e!=0)?(RET_ERRNO(e)):RET_ERROR;
if( err == 0 || err == -EINVAL )
r = RET_ERROR;
else
r = RET_ERRNO(err);
r2 = uncompress_error(file);
RET_UPDATE(r, r2);
uncompress_abort(file);
return r;
}
archive_read_finish(tar);
*found_p = false;
return RET_OK;
return uncompress_close(file);
}
#endif
......@@ -564,10 +633,16 @@ retvalue sourceextraction_analyse(struct sourceextraction *e, const char *fullfi
return RET_NOTHING;
}
r = parsediff(f, e->section_p, e->priority_p, &found);
if( RET_IS_OK(r) )
r = uncompress_close(f);
else
(void)uncompress_close(f);
if( RET_IS_OK(r) ) {
if( !found )
r = uncompress_close(f);
else {
r = uncompress_error(f);
uncompress_abort(f);
}
} else {
uncompress_abort(f);
}
if( !RET_IS_OK(r) )
e->failed = true;
else if( found )
......
......@@ -136,6 +136,10 @@ echo 2.0 > debian-binary
dodo ar qcfS fake.deb debian-binary control.tar.bz2 data.tar.lzma
rm *.tar.bz2 *.tar.lzma debian-binary
# TODO: there could be a problem here with .deb files that have data after the
# ./control file in data.tar and using an external uncompressor.
# But how to test this when there is no way to trigger it in the default built?
testrun - __extractcontrol fake.deb 3<<EOF
stdout
*=Package: fake
......@@ -151,5 +155,89 @@ stdout
*=/testfile.lzma
EOF
rm testfile* smallfile* fake.deb
rm fake.deb
# Now check extracting Section/Priority from an .dsc
mkdir debian
cat > debian/control <<EOF
Package: fake
Maintainer: Me
Section: admin
Priority: extra
Package: abinary
Architecture: all
EOF
echo generating fake dirs
for n in $(seq 100000) ; do echo "/$n" ; done > debian/dirs
dd if=/dev/zero of=debian/zzz bs=1024 count=4096
tar -cf - debian | lzma > fake_1-1.debian.tar.lzma
mkdir fake-1
mkdir fake-1.orig
cp -al debian fake-1/debian
cp -al debian fake-1.orig/debian
sed -e 's/1/2/' fake-1/debian/dirs > fake-1/debian.dirs.new
mv fake-1/debian.dirs.new fake-1/debian/dirs
diff -ruN fake-1.orig fake-1 | lzma > fake_1-1.diff.lzma
rm -r debian
# .debian.tar and .diff usually do not happen at the same time, but easier testing...
cat > fake_1-1.dsc << EOF
Format: 3.0
Source: fake
Binary: abinary
Architecture: all
Version: 17
Maintainer: Me
Files:
$(mdandsize fake_1-1.diff.lzma) fake_1-1.diff.lzma
$(mdandsize fake_1-1.debian.tar.lzma) fake_1-1.debian.tar.lzma
00000000000000000000000000000000 0 fake_1.orig.tar.lzma
EOF
testrun - __extractsourcesection fake_1-1.dsc 3<<EOF
=Data seems not to be signed trying to use directly...
stdout
*=Section: admin
*=Priority: extra
EOF
# now check only partial reading of the .diff:
rm fake-1/debian/control
cat > fake-1/debian/control <<EOF
Package: fake
Maintainer: MeToo
Section: base
Priority: required
Package: abinary
Architecture: all
EOF
diff -ruN fake-1.orig fake-1 | lzma > fake_1-1.diff.lzma
rm -r fake-1 fake-1.orig
cat > fake_1-1.dsc << EOF
Format: 3.0
Source: fake
Binary: abinary
Architecture: all
Version: 17
Maintainer: Me
Files:
$(mdandsize fake_1-1.diff.lzma) fake_1-1.diff.lzma
$(mdandsize fake_1-1.debian.tar.lzma) fake_1-1.debian.tar.lzma
00000000000000000000000000000000 0 fake_1.orig.tar.lzma
EOF
testrun - __extractsourcesection fake_1-1.dsc 3<<EOF
=Data seems not to be signed trying to use directly...
stdout
*=Section: base
*=Priority: required
EOF
rm testfile* smallfile*
# TODO: test error messages of truncated files and files with errors in the
# compressed stream...
testsuccess
......@@ -356,7 +356,7 @@ static inline retvalue builtin_uncompress(const char *compressed, const char *de
e = errno;
fprintf(stderr, "Error %d creating '%s': %s\n",
e, destination, strerror(e));
(void)uncompress_close(f);
uncompress_abort(f);
return RET_ERRNO(e);
}
do {
......@@ -373,7 +373,7 @@ static inline retvalue builtin_uncompress(const char *compressed, const char *de
fprintf(stderr,
"Error %d writing to '%s': %s\n", e, destination, strerror(e) );
close(destfd);
(void)uncompress_close(f);
uncompress_abort(f);
return RET_ERRNO(e);
}
bytes_written += written;
......@@ -936,6 +936,152 @@ static retvalue uncompress_commonclose(struct compressedfile *file, int *errno_p
/* not reached */
}
/* check if there has been an error yet for this stream */
retvalue uncompress_error(struct compressedfile *file) {
int e, zerror, status;
const char *msg;
pid_t pid;
if( file == NULL )
return RET_NOTHING;
switch( file->compression ) {
case c_none:
if( file->error == 0 )
return RET_OK;
break;
case c_gzip:
msg = gzerror(file->gz, &zerror);
if( zerror == 0 )
return RET_OK;
if( zerror != Z_ERRNO ) {
fprintf(stderr, "Zlib error %d uncompressing file '%s': %s\n",
zerror,
file->filename,
msg);
return RET_ERROR_Z;
}
break;
#ifdef HAVE_LIBBZ2
case c_bzip2:
msg = BZ2_bzerror(file->bz, &zerror);
if( zerror < 0 ) {
fprintf(stderr, "libbz2 error %d uncompressing file '%s': %s\n",
zerror,
file->filename,
msg);
return RET_ERROR_BZ2;
} else
return RET_OK;
#endif
default:
if( file->error != 0 )
break;
if( file->pid <= 0 )
return RET_OK;
pid = waitpid(file->pid, &status, WNOHANG);
if( pid < 0 ) {
e = errno;
fprintf(stderr, "Error looking for child %lu (a '%s'): %s\n",
(long unsigned)file->pid,
extern_uncompressors[file->compression],
strerror(e));
return RET_ERRNO(e);
}
if( pid != file->pid ) {
/* still running */
return RET_OK;
}
file->pid = -1;
if( WIFEXITED(status) ) {
if( WEXITSTATUS(status) == 0 )
return RET_OK;
else {
fprintf(stderr,
"%s exited with code %d\n",
extern_uncompressors[file->compression],
(int)(WEXITSTATUS(status)));
return RET_ERROR;
}
} else if( WIFSIGNALED(status) && WTERMSIG(status) != SIGUSR2 ) {
fprintf(stderr,
"%s killed by signal %d\n",
extern_uncompressors[file->compression],
(int)(WTERMSIG(status)));
return RET_ERROR;
} else {
fprintf(stderr,
"%s failed\n",
extern_uncompressors[file->compression]);
return RET_ERROR;
}
}
/* an error, but which? */
if( file->error != 0 ) {
fprintf(stderr, "Error %d uncompressing file '%s': %s\n",
file->error, file->filename,
strerror(file->error));
return RET_ERRNO(file->error);
} else
return RET_ERROR;
}
void uncompress_abort(struct compressedfile *file) {
pid_t pid;
int e, status;
if( file == NULL )
return;
switch( file->compression ) {
case c_none:
if( file->fd >= 0 )
(void)close(file->fd);
break;
case c_gzip:
(void)gzclose(file->gz);
break;
#ifdef HAVE_LIBBZ2
case c_bzip2:
BZ2_bzclose(file->bz);
break;
#endif
default:
/* kill before closing, to avoid it getting
* a sigpipe */
if( file->pid > 0 )
kill(file->pid, SIGTERM);
if( file->infd >= 0 )
(void)close(file->infd);
if( file->fd >= 0 )
(void)close(file->fd);
if( file->pipeinfd != -1 )
(void)close(file->pipeinfd);
do {
pid = waitpid(file->pid, &status, 0);
e = errno;
if( interrupted() ) {
break;
}
} while( pid == -1 && (e == EINTR || e == EAGAIN) );
if( pid == -1 )
break;
if( WIFEXITED(status) ) {
break;
} else if( WIFSIGNALED(status)
&& WTERMSIG(status) != SIGTERM
&& WTERMSIG(status) != SIGUSR2 ) {
fprintf(stderr, "%s killed by signal %d",
extern_uncompressors[file->compression],
(int)(WTERMSIG(status)));
}
}
free(file->filename);
free(file);
}
retvalue uncompress_fdclose(struct compressedfile *file, int *errno_p, const char **msg_p) {
retvalue result;
......@@ -984,4 +1130,3 @@ retvalue uncompress_close(struct compressedfile *file) {
free(file);
return r;
}
......@@ -40,6 +40,8 @@ struct compressedfile;
retvalue uncompress_open(/*@out@*/struct compressedfile **, const char *, enum compression);
int uncompress_read(struct compressedfile *, void *buffer, int);
retvalue uncompress_error(/*@const@*/struct compressedfile *);
void uncompress_abort(/*@only@*/struct compressedfile *);
retvalue uncompress_close(/*@only@*/struct compressedfile *);
retvalue uncompress_fdclose(/*@only@*/struct compressedfile *, int *, const char **);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment