Commit dd84b0bd authored by Bernhard Link's avatar Bernhard Link

unify uncompression, adding support for extraction section and priority from a...

unify uncompression, adding support for extraction section and priority from a diff compressed with any supported compression (adds .bz2 currently) and restoring from snapshots with only .bz2 indices
parent bb494d35
2008-08-24
* unify reading of compressed files, adding support for:
- extracting section and priority from a .diff.bz2
- restoring from a snapshot with only .bz2 indices
2008-08-23
* massive refactorisation of the update code to retrieve
remote index files. Most important modifications:
......
......@@ -18,12 +18,12 @@ AM_CPPFLAGS = -std=gnu99 -Wall $(ARCHIVECPP) $(DBCPPFLAGS)
reprepro_LDADD = $(ARCHIVELIBS) $(DBLIBS)
changestool_LDADD = $(ARCHIVELIBS)
reprepro_SOURCES = uncompression.c remoterepository.c indexfile.c copypackages.c readcompressed.c sourceextraction.c checksums.c readtextfile.c filecntl.c sha1.c sha256.c configparser.c database.c freespace.c log.c changes.c incoming.c uploaderslist.c guesscomponent.c files.c md5.c dirs.c chunks.c reference.c binaries.c sources.c checks.c names.c dpkgversions.c release.c mprintf.c updates.c strlist.c signature.c distribution.c checkindeb.c checkindsc.c checkin.c upgradelist.c target.c aptmethod.c downloadcache.c main.c override.c terms.c ignore.c filterlist.c exports.c tracking.c optionsfile.c readrelease.c donefile.c pull.c contents.c filelist.c $(ARCHIVE_USED) $(ARCHIVE_CONTENTS)
reprepro_SOURCES = uncompression.c remoterepository.c indexfile.c copypackages.c sourceextraction.c checksums.c readtextfile.c filecntl.c sha1.c sha256.c configparser.c database.c freespace.c log.c changes.c incoming.c uploaderslist.c guesscomponent.c files.c md5.c dirs.c chunks.c reference.c binaries.c sources.c checks.c names.c dpkgversions.c release.c mprintf.c updates.c strlist.c signature.c distribution.c checkindeb.c checkindsc.c checkin.c upgradelist.c target.c aptmethod.c downloadcache.c main.c override.c terms.c ignore.c filterlist.c exports.c tracking.c optionsfile.c readrelease.c donefile.c pull.c contents.c filelist.c $(ARCHIVE_USED) $(ARCHIVE_CONTENTS)
EXTRA_reprepro_SOURCE = $(ARCHIVE_UNUSED)
changestool_SOURCES = readcompressed.c sourceextraction.c readtextfile.c filecntl.c tool.c chunkedit.c strlist.c checksums.c sha1.c sha256.c md5.c mprintf.c chunks.c signature.c dirs.c names.c $(ARCHIVE_USED)
changestool_SOURCES = uncompression.c sourceextraction.c readtextfile.c filecntl.c tool.c chunkedit.c strlist.c checksums.c sha1.c sha256.c md5.c mprintf.c chunks.c signature.c dirs.c names.c $(ARCHIVE_USED)
noinst_HEADERS = uncompression.h remoterepository.h copypackages.h readcompressed.h sourceextraction.h checksums.h readtextfile.h filecntl.h sha1.h sha256.h configparser.h database_p.h database.h freespace.h log.h changes.h incoming.h guesscomponent.h md5.h dirs.h files.h chunks.h reference.h binaries.h sources.h checks.h names.h release.h error.h mprintf.h updates.h strlist.h signature.h distribution.h debfile.h checkindeb.h checkindsc.h upgradelist.h target.h aptmethod.h downloadcache.h override.h terms.h ignore.h filterlist.h dpkgversions.h checkin.h exports.h globals.h tracking.h trackingt.h optionsfile.h readrelease.h donefile.h pull.h ar.h filelist.h contents.h chunkedit.h uploaderslist.h indexfile.h
noinst_HEADERS = uncompression.h remoterepository.h copypackages.h sourceextraction.h checksums.h readtextfile.h filecntl.h sha1.h sha256.h configparser.h database_p.h database.h freespace.h log.h changes.h incoming.h guesscomponent.h md5.h dirs.h files.h chunks.h reference.h binaries.h sources.h checks.h names.h release.h error.h mprintf.h updates.h strlist.h signature.h distribution.h debfile.h checkindeb.h checkindsc.h upgradelist.h target.h aptmethod.h downloadcache.h override.h terms.h ignore.h filterlist.h dpkgversions.h checkin.h exports.h globals.h tracking.h trackingt.h optionsfile.h readrelease.h donefile.h pull.h ar.h filelist.h contents.h chunkedit.h uploaderslist.h indexfile.h
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in $(srcdir)/configure $(srcdir)/stamp-h.in $(srcdir)/aclocal.m4 $(srcdir)/config.h.in
......
......@@ -673,7 +673,7 @@ retvalue copy_from_file(struct database *database, struct distribution *into, co
into->codename);
return RET_ERROR;
}
result = indexfile_open(&i, filename);
result = indexfile_open(&i, filename, c_none);
if( !RET_IS_OK(result) )
return result;
result = RET_NOTHING;
......@@ -704,6 +704,7 @@ static retvalue restore_from_snapshot(struct database *database, struct distribu
struct package_list list;
struct target *target;
char *basedir;
enum compression compression;
basedir = calc_snapshotbasedir(into->codename, snapshotname);
if( FAILEDTOALLOC(basedir) )
......@@ -726,16 +727,28 @@ static retvalue restore_from_snapshot(struct database *database, struct distribu
* how the file is named now and try all readable
* compressions */
compression = c_none;
filename = calc_dirconcat3(
basedir, target->relativedirectory,
target->exportmode->filename);
if( filename != NULL && !isregularfile(filename) ) {
/* no uncompressed file found, try .gz */
free(filename);
compression = c_gzip;
filename = mprintf("%s/%s/%s.gz",
basedir, target->relativedirectory,
target->exportmode->filename);
}
#ifdef HAVE_LIBBZ2
if( filename != NULL && !isregularfile(filename) ) {
/* no uncompressed or .gz file found, try .bz2 */
free(filename);
compression = c_bzip2;
filename = mprintf("%s/%s/%s.bz2",
basedir, target->relativedirectory,
target->exportmode->filename);
}
#endif
if( filename != NULL && !isregularfile(filename) ) {
free(filename);
fprintf(stderr,
......@@ -751,7 +764,7 @@ static retvalue restore_from_snapshot(struct database *database, struct distribu
result = RET_ERROR_OOM;
break;
}
result = indexfile_open(&i, filename);
result = indexfile_open(&i, filename, compression);
if( !RET_IS_OK(result) )
break;
while( indexfile_getnext(i, &packagename, &version, &control,
......
......@@ -21,18 +21,18 @@
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include <zlib.h>
#include <assert.h>
#include "error.h"
#include "chunks.h"
#include "names.h"
#include "uncompression.h"
#include "indexfile.h"
/* the purpose of this code is to read index files, either from a snapshot
* previously generated or downloaded while updating. */
struct indexfile {
gzFile f;
struct compressedfile *f;
char *filename;
int linenumber, startlinenumber;
retvalue status;
......@@ -43,8 +43,9 @@ struct indexfile {
extern int verbose;
retvalue indexfile_open(struct indexfile **file_p, const char *filename) {
retvalue indexfile_open(struct indexfile **file_p, const char *filename, enum compression compression) {
struct indexfile *f = calloc(1, sizeof(struct indexfile));
retvalue r;
if( FAILEDTOALLOC(f) )
return RET_ERROR_OOM;
......@@ -53,10 +54,9 @@ retvalue indexfile_open(struct indexfile **file_p, const char *filename) {
free(f);
return RET_ERROR_OOM;
}
f->f = gzopen(filename, "r");
if( f->f == NULL ) {
fprintf(stderr, "Unable to open file %s: %s\n",
filename, strerror(errno));
r = uncompress_open(&f->f, filename, compression);
assert( r != RET_NOTHING );
if( RET_WAS_ERROR(r) ) {
free(f->filename);
free(f);
return RET_ERRNO(errno);
......@@ -70,7 +70,8 @@ retvalue indexfile_open(struct indexfile **file_p, const char *filename) {
/* +1 for *d = '\0' in eof case */
f->buffer = malloc(f->size + 1);
if( FAILEDTOALLOC(f->buffer) ) {
(void)gzclose(f->f);
(void)uncompress_close(f->f);
free(f->filename);
free(f);
return RET_ERROR_OOM;
}
......@@ -81,12 +82,11 @@ retvalue indexfile_open(struct indexfile **file_p, const char *filename) {
retvalue indexfile_close(struct indexfile *f) {
retvalue r;
//TODO: check result:
gzclose(f->f);
r = uncompress_close(f->f);
free(f->filename);
free(f->buffer);
r = f->status;
RET_UPDATE(r, f->status);
free(f);
return r;
......@@ -166,7 +166,7 @@ static retvalue indexfile_get(struct indexfile *f) {
return RET_ERROR;
}
bytes_read = gzread(f->f, d, f->size - f->ofs);
bytes_read = uncompress_read(f->f, d, f->size - f->ofs);
if( bytes_read < 0 )
return RET_ERROR;
else if( bytes_read == 0 )
......
......@@ -11,7 +11,7 @@
struct indexfile;
retvalue indexfile_open(/*@out@*/struct indexfile **, const char *);
retvalue indexfile_open(/*@out@*/struct indexfile **, const char *, enum compression);
retvalue indexfile_close(/*@only@*/struct indexfile *);
bool indexfile_getnext(struct indexfile *, /*@out@*/char **, /*@out@*/char **, /*@out@*/const char **, const struct target *, bool allowwrongarchitecture);
......
/* This file is part of "reprepro"
* Copyright (C) 2008 Bernhard R. Link
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA
*/
#include <config.h>
#include <errno.h>
#include <assert.h>
#include <limits.h>
#include <malloc.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <zlib.h>
#ifdef HAVE_LIBBZ2
#include <bzlib.h>
#endif
#include "error.h"
#include "readcompressed.h"
#define c_uncompressed c_none
#define c_gzipped c_gzip
extern int verbose;
struct readcompressed {
// preliminary, as dead slow...
gzFile f;
char buffer[4097];
size_t lineno;
};
bool readcompressed_getline(struct readcompressed *r, /*@out@*/const char **line) {
char *c;
r->lineno++;
if( gzgets(r->f, r->buffer, 4096) == NULL )
return false;
r->buffer[4096] = '\0';
c = strchr(r->buffer, '\n');
if( c == NULL )
return false;
*c = '\0';
while( --c > r->buffer && *c == '\r' )
*c = '\0';
*line = r->buffer;
return true;
}
char readcompressed_overlinegetchar(struct readcompressed *r) {
char *c, ch;
r->lineno++;
if( gzgets(r->f, r->buffer, 4096) == NULL )
return false;
r->buffer[4096] = '\0';
ch = r->buffer[0];
if( ch == '\n' ) {
return '\0';
}
c = strchr(r->buffer, '\n');
while( c == NULL ) {
if( gzgets(r->f, r->buffer, 4096) == NULL )
return ch;
c = strchr(r->buffer, '\n');
}
return ch;
}
retvalue readcompressed_open(/*@out@*/struct readcompressed **r, const char *filename, enum compression c) {
struct readcompressed *n;
if( c != c_uncompressed && c != c_gzipped )
return RET_NOTHING;
n = calloc(1, sizeof(struct readcompressed));
if( n == NULL )
return RET_ERROR_OOM;
n->f = gzopen(filename, "r");
if( n->f == NULL ) {
free(n);
return RET_NOTHING;
}
*r = n;
return RET_OK;
}
retvalue readcompressed_close(struct readcompressed *r) {
if( r != NULL ) {
gzclose(r->f);
free(r);
}
// TODO: implement error handling...
return RET_OK;
}
void readcompressed_abort(struct readcompressed *r) {
if( r != NULL ) {
if( verbose > 20 ) {
fprintf(stderr, "Aborted reading compessed file at line %llu\n",
(long long unsigned int)r->lineno);
}
gzclose(r->f);
free(r);
}
}
#ifndef REPREPRO_READCOMPRESSED_H
#define REPREPRO_READCOMPRESSED_H
// TODO: integrate this into uncompress.c...
#define unsupportedcompression(x) ( x >= c_bzip2 )
struct readcompressed;
bool readcompressed_getline(struct readcompressed *, /*@out@*/const char **);
char readcompressed_overlinegetchar(struct readcompressed *);
retvalue readcompressed_open(/*@out@*/struct readcompressed **, const char *, enum compression);
retvalue readcompressed_close(/*@only@*/struct readcompressed *);
void readcompressed_abort(/*@only@*/struct readcompressed *);
#endif
......@@ -30,7 +30,7 @@
#include "error.h"
#include "filecntl.h"
#include "chunks.h"
#include "readcompressed.h"
#include "uncompression.h"
#include "sourceextraction.h"
struct sourceextraction {
......@@ -106,7 +106,7 @@ bool sourceextraction_needs(struct sourceextraction *e, int *ofs_p) {
if( e->failed || e->completed )
return false;
if( e->difffile >= 0 ) {
if( unsupportedcompression(e->diffcompression) )
if( !uncompression_supported(e->diffcompression) )
// TODO: errormessage
return false;
*ofs_p = e->difffile;
......@@ -130,20 +130,96 @@ bool sourceextraction_needs(struct sourceextraction *e, int *ofs_p) {
return false;
}
static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p, /*@null@*/char **priority_p, bool *found_p) {
static retvalue parsediff(struct compressedfile *f, /*@null@*/char **section_p, /*@null@*/char **priority_p, bool *found_p) {
size_t destlength, lines_in, lines_out;
const char *p, *s;
#define BUFSIZE 4096
char buffer[BUFSIZE];
int bytes_read, used = 0, filled = 0;
inline bool u_getline(void);
inline bool u_getline(void) {
do {
if( filled - used > 0 ) {
char *n;
p = buffer + used;
n = memchr(p, '\n', filled - used);
if( n != NULL ) {
used += 1 + (n - p);
*n = '\0';
while( --n >= p && *n == '\r' )
*n = '\0';
return true;
}
} else { assert( filled == used );
filled = 0;
used = 0;
}
if( filled == BUFSIZE ) {
if( used == 0 )
/* overlong line */
return false;
memmove(buffer, buffer + used, filled - used);
filled -= used;
used = 0;
}
bytes_read = uncompress_read(f, buffer + filled, BUFSIZE - filled);
if( bytes_read <= 0 )
return false;
filled += bytes_read;
} while( true );
}
inline char u_overlinegetchar(void);
inline char u_overlinegetchar(void) {
const char *n;
char ch;
if( filled - used > 0 ) {
ch = buffer[used];
} else { assert( filled == used );
used = 0;
bytes_read = uncompress_read(f, buffer, BUFSIZE);
if( bytes_read <= 0 ) {
filled = 0;
return '\0';
}
filled = bytes_read;
ch = buffer[0];
}
if( ch == '\n' )
return '\0';
/* over rest of the line */
n = memchr(buffer + used, '\n', filled - used);
if( n != NULL ) {
used = 1 + (n - buffer);
return ch;
}
used = 0;
filled = 0;
/* need to read more to get to the end of the line */
do { /* these lines can be long */
bytes_read = uncompress_read(f, buffer, BUFSIZE);
if( bytes_read <= 0 )
return false;
n = memchr(buffer, '\n', bytes_read);
} while( n == NULL );
used = 1 + (n - buffer);
filled = bytes_read;
return ch;
}
/* we are assuming the exact format dpkg-source generates here... */
if( !readcompressed_getline(f, &p) ) {
/* empty file */
if( !u_getline() ) {
/* empty or strange file */
*found_p = false;
return RET_OK;
}
if( unlikely(memcmp(p, "--- ", 4) != 0) )
return RET_NOTHING;
if( !readcompressed_getline(f, &p) )
if( !u_getline() )
/* so short a file? */
return RET_NOTHING;
if( unlikely(memcmp(p, "+++ ", 4) != 0) )
......@@ -157,7 +233,7 @@ static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p,
while( strcmp(s, "debian/control") != 0 ) {
if( unlikely(interrupted()) )
return RET_ERROR_INTERRUPTED;
if( !readcompressed_getline(f, &p) )
if( !u_getline() )
return RET_NOTHING;
while( memcmp(p, "@@ -", 4) == 0) {
if( unlikely(interrupted()) )
......@@ -197,7 +273,7 @@ static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p,
while( lines_in > 0 || lines_out > 0 ) {
char ch;
ch = readcompressed_overlinegetchar(f);
ch = u_overlinegetchar();
switch( ch ) {
case '+':
if( unlikely(lines_out == 0) )
......@@ -208,6 +284,7 @@ static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p,
if( unlikely(lines_out == 0) )
return RET_NOTHING;
lines_out--;
/* no break */
case '-':
if( unlikely(lines_in == 0) )
return RET_NOTHING;
......@@ -217,7 +294,7 @@ static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p,
return RET_NOTHING;
}
}
if( !readcompressed_getline(f, &p) ) {
if( !u_getline() ) {
*found_p = false;
/* nothing found successfully */
return RET_OK;
......@@ -225,7 +302,7 @@ static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p,
}
if( unlikely(memcmp(p, "--- ", 4) != 0) )
return RET_NOTHING;
if( !readcompressed_getline(f, &p) )
if( !u_getline() )
return RET_NOTHING;
if( unlikely(memcmp(p, "+++ ", 4) != 0) )
return RET_NOTHING;
......@@ -235,7 +312,7 @@ static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p,
s++;
}
/* found debian/control */
if( !readcompressed_getline(f, &p) )
if( !u_getline() )
return RET_NOTHING;
if( unlikely(memcmp(p, "@@ -", 4) != 0) )
return RET_NOTHING;
......@@ -269,7 +346,7 @@ static retvalue parsediff(struct readcompressed *f, /*@null@*/char **section_p,
while( lines_out > 0 ) {
if( unlikely(interrupted()) )
return RET_ERROR_INTERRUPTED;
if( !readcompressed_getline(f, &p) )
if( !u_getline() )
return RET_NOTHING;
switch( *(p++) ) {
......@@ -467,21 +544,22 @@ retvalue sourceextraction_analyse(struct sourceextraction *e, const char *fullfi
assert( e->difffile >= 0 );
#endif
if( e->difffile >= 0 ) {
struct readcompressed *f;
struct compressedfile *f;
assert( !unsupportedcompression(e->diffcompression) );
assert( uncompression_supported(e->diffcompression) );
e->difffile = -1;
r = readcompressed_open(&f, fullfilename, e->diffcompression);
r = uncompress_open(&f, fullfilename, e->diffcompression);
if( !RET_IS_OK(r) ) {
e->failed = true;
return r;
/* being unable to read a file is no hard error... */
return RET_NOTHING;
}
r = parsediff(f, e->section_p, e->priority_p, &found);
if( RET_IS_OK(r) )
r = readcompressed_close(f);
r = uncompress_close(f);
else
readcompressed_abort(f);
(void)uncompress_close(f);
if( !RET_IS_OK(r) )
e->failed = true;
else if( found )
......
This diff is collapsed.
......@@ -3,16 +3,24 @@
/* "", ".gz", ... */
extern const char * const uncompression_suffix[c_COUNT];
extern /*@null@*/ char *extern_uncompressors[c_COUNT];
/* there are two different modes: uncompress a file to memory,
* or uncompress (possibly multiple files) on the filesystem,
* controled by aptmethods */
#ifdef HAVE_LIBBZ2
#define uncompression_supported(c) ((c) == c_gzip || (c) == c_none || (c) == c_bzip2)
#define ONLYWITHBZ2(a) a
#else
#define uncompression_supported(c) ((c) == c_gzip || (c) == c_none)
#define ONLYWITHBZ2(a)
#endif
#define uncompression_supported(c) ( \
ONLYWITHBZ2( (c) == c_bzip2 || ) \
(c) == c_gzip || \
(c) == c_none || \
extern_uncompressors[c] != NULL)
/**** functions for aptmethod.c ****/
/* we got an pid, check if it is a uncompressor we care for */
retvalue uncompress_checkpid(pid_t, int);
......@@ -23,8 +31,21 @@ typedef retvalue finishaction(void *, const char *, bool);
/* uncompress and call action when finished */
retvalue uncompress_queue_file(const char *, const char *, enum compression, finishaction *, void *, bool *);
/**** functions for update.c (uncompressing an earlier downloaded file) ****/
retvalue uncompress_file(const char *, const char *, enum compression);
/**** functions for indexfile.c (uncompressing to memory) ****/
// and perhaps also sourceextraction.c
struct compressedfile;
retvalue uncompress_open(/*@out@*/struct compressedfile **, const char *, enum compression);
int uncompress_read(struct compressedfile *, void *buffer, int);
retvalue uncompress_close(/*@only@*/struct compressedfile *);
/**** general initialisation ****/
/* check for existance of external programs */
void uncompressions_check(void);
......
......@@ -437,7 +437,7 @@ retvalue upgradelist_update(struct upgradelist *upgrade, struct aptmethod *metho
const char *control;
retvalue result, r;
r = indexfile_open(&i, filename);
r = indexfile_open(&i, filename, c_none);
if( !RET_IS_OK(r) )
return r;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment