Commit ab6ebd7d authored by Bernhard Link's avatar Bernhard Link

refactor update code into remoterepository.c, improving avoiding multiple...

refactor update code into remoterepository.c, improving avoiding multiple downloads and laying foundation for many new features to come
parent 82ab6225
2008-08-23
* massive refactorisation of the update code to retrieve
remote index files. Most important modifications:
- when the same remote distribution is needed by multiple
updates, then the index files are only downloaded once.
(still needs futher changes to allow better detection
of the same source).
- ListHooks are called once per use (should mostly only
make a difference for flat sources or with settings
where this is needed).
- --nolistsdownload not only not downloads lists and has
no other effects (checksums still checked, --noskipold
no longer implied).
- deleting of old no longer needed lists (the default
--nokeepunneeded) no longer exists.
- index files are stored uncompressed in lists/ and the
way files are named there is less strange...
- many other changes are possible now and will hopefully
be implemented soon.
2008-08-22
* some internal cleanup preparing for future changes...
......
......@@ -18,12 +18,12 @@ AM_CPPFLAGS = -std=gnu99 -Wall $(ARCHIVECPP) $(DBCPPFLAGS)
reprepro_LDADD = $(ARCHIVELIBS) $(DBLIBS)
changestool_LDADD = $(ARCHIVELIBS)
reprepro_SOURCES = uncompression.c indexfile.c copypackages.c readcompressed.c sourceextraction.c checksums.c readtextfile.c filecntl.c sha1.c sha256.c configparser.c database.c freespace.c log.c changes.c incoming.c uploaderslist.c guesscomponent.c files.c md5.c dirs.c chunks.c reference.c binaries.c sources.c checks.c names.c dpkgversions.c release.c mprintf.c updates.c strlist.c signature.c distribution.c checkindeb.c checkindsc.c checkin.c upgradelist.c target.c aptmethod.c downloadcache.c main.c override.c terms.c ignore.c filterlist.c exports.c tracking.c optionsfile.c readrelease.c donefile.c pull.c contents.c filelist.c $(ARCHIVE_USED) $(ARCHIVE_CONTENTS)
reprepro_SOURCES = uncompression.c remoterepository.c indexfile.c copypackages.c readcompressed.c sourceextraction.c checksums.c readtextfile.c filecntl.c sha1.c sha256.c configparser.c database.c freespace.c log.c changes.c incoming.c uploaderslist.c guesscomponent.c files.c md5.c dirs.c chunks.c reference.c binaries.c sources.c checks.c names.c dpkgversions.c release.c mprintf.c updates.c strlist.c signature.c distribution.c checkindeb.c checkindsc.c checkin.c upgradelist.c target.c aptmethod.c downloadcache.c main.c override.c terms.c ignore.c filterlist.c exports.c tracking.c optionsfile.c readrelease.c donefile.c pull.c contents.c filelist.c $(ARCHIVE_USED) $(ARCHIVE_CONTENTS)
EXTRA_reprepro_SOURCE = $(ARCHIVE_UNUSED)
changestool_SOURCES = readcompressed.c sourceextraction.c readtextfile.c filecntl.c tool.c chunkedit.c strlist.c checksums.c sha1.c sha256.c md5.c mprintf.c chunks.c signature.c dirs.c names.c $(ARCHIVE_USED)
noinst_HEADERS = uncompression.h copypackages.h readcompressed.h sourceextraction.h checksums.h readtextfile.h filecntl.h sha1.h sha256.h configparser.h database_p.h database.h freespace.h log.h changes.h incoming.h guesscomponent.h md5.h dirs.h files.h chunks.h reference.h binaries.h sources.h checks.h names.h release.h error.h mprintf.h updates.h strlist.h signature.h distribution.h debfile.h checkindeb.h checkindsc.h upgradelist.h target.h aptmethod.h downloadcache.h override.h terms.h ignore.h filterlist.h dpkgversions.h checkin.h exports.h globals.h tracking.h trackingt.h optionsfile.h readrelease.h donefile.h pull.h ar.h filelist.h contents.h chunkedit.h uploaderslist.h indexfile.h
noinst_HEADERS = uncompression.h remoterepository.h copypackages.h readcompressed.h sourceextraction.h checksums.h readtextfile.h filecntl.h sha1.h sha256.h configparser.h database_p.h database.h freespace.h log.h changes.h incoming.h guesscomponent.h md5.h dirs.h files.h chunks.h reference.h binaries.h sources.h checks.h names.h release.h error.h mprintf.h updates.h strlist.h signature.h distribution.h debfile.h checkindeb.h checkindsc.h upgradelist.h target.h aptmethod.h downloadcache.h override.h terms.h ignore.h filterlist.h dpkgversions.h checkin.h exports.h globals.h tracking.h trackingt.h optionsfile.h readrelease.h donefile.h pull.h ar.h filelist.h contents.h chunkedit.h uploaderslist.h indexfile.h
MAINTAINERCLEANFILES = $(srcdir)/Makefile.in $(srcdir)/configure $(srcdir)/stamp-h.in $(srcdir)/aclocal.m4 $(srcdir)/config.h.in
......
......@@ -1190,9 +1190,10 @@ static retvalue senddata(struct aptmethod *method) {
assert( strchr(todo->uri, '\n') == NULL &&
strchr(todo->filename,'\n') == NULL );
/* http-aptmethod seems to loose the last byte if the file is already
* in place, so we better unlink the target first... */
if( todo->compression == c_none )
unlink(todo->filename);
* in place, so we better unlink the target first...
* but this is done elsewhere already
unlink(todo->filename);
*/
method->command = mprintf(
"600 URI Acquire\nURI: %s\nFilename: %s%s\n\n",
todo->uri, todo->filename,
......
......@@ -120,7 +120,7 @@ retvalue distribution_remove_packages(struct distribution *, struct database *,
/*@dependent@*/struct target *distribution_getpart(const struct distribution *distribution,const char *component,const char *architecture,const char *packagetype);
/* like distribtion_getpart, but returns NULL if there is no such target */
/*@dependent@*/struct target *distribution_gettarget(const struct distribution *distribution,const char *component,const char *architecture,const char *packagetype);
/*@null@*//*@dependent@*/struct target *distribution_gettarget(const struct distribution *distribution,const char *component,const char *architecture,const char *packagetype);
retvalue distribution_fullexport(struct distribution *distribution, struct database *);
......
......@@ -189,9 +189,12 @@ Ignore errors of type \fIwhat\fP. See the section \fBERROR IGNORING\fP
for possible values.
.TP
.B \-\-nolistsdownload
When running \fBupdate\fP or \fBcheckupdate\fP do not download any Release
or index files (and also do not check them). This is hardly useful except
when you just run one of those command for the same distributions.
When running \fBupdate\fP, \fBcheckupdate\fP or \fBpredelete\fP do not download
any Release or index files.
This is hardly useful except when you just run one of those
command for the same distributions.
And even then reprepro is usually good in
not downloading except \fBRelease\fP and \fBRelease.gpg\fP files again.
.TP
.B \-\-keepunreferencedfiles
Do not delete files that are no longer used because the package they
......@@ -374,6 +377,9 @@ Delete all files in \fIlistdir\fP (default \fIbasedir\fP\fB/lists\fP) that do no
belong to any update rule for any distribution.
I.e. all files are deleted in that directory that no \fbupdate\fP
command in the current configuration can use.
(The files are usually left there, so if they are needed again they
do not need to be downloaded again. Though in many easy cases not
even those files will be needed.)
.TP
.BR pull " [ " \fIcodenames\fP " ]"
pull in newer packages into the specified distributions (all if none given)
......@@ -1088,6 +1094,14 @@ If this is given, it is executed for all downloaded index files
with the downloaded list as first and a filename that will
be used instead of this. (e.g. "ListHook: /bin/cp" works
but does nothing.)
If a file will be read multiple times, it is processed multiple
times, with the environment variables
.BR REPREPRO_FILTER_CODENAME ", " REPREPRO_FILTER_PACKAGETYPE ", "
.BR REPREPRO_FILTER_COMPONENT " and " REPREPRO_FILTER_ARCHITECTURE
set to the where this file will be added and
.B REPREPRO_FILTER_PATTERN
to the name of the update rule causing it.
.SS conf/pulls
This file contains the rules for pulling packages from one
distribution to another.
......
/* This file is part of "reprepro"
* Copyright (C) 2006,2007 Bernhard R. Link
* Copyright (C) 2008 Bernhard R. Link
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
......@@ -27,116 +27,215 @@
#include "donefile.h"
#include "names.h"
#include "checksums.h"
#include "remoterepository.h"
retvalue donefile_isold(const char *filename, const struct checksums *expected) {
char buffer[200];
const char *start;
size_t len;
ssize_t bytes;
int fd;
char *donefilename = calc_addsuffix(filename,"done");
if( donefilename == NULL )
/* This stores what an distribution that is updated from remote repositories
* has already processed, so that things already processed do not have to be
* downloaded or processed again. */
struct markdonefile {
char *finalfilename;
char *tempfilename;
FILE *file;
};
static inline char *donefilename(const char *codename) {
return genlistsfilename("lastseen", 2, "", codename, NULL);
}
retvalue markdone_create(const char *codename, struct markdonefile **done_p) {
struct markdonefile *done;
done = malloc(sizeof(struct markdonefile));
if( FAILEDTOALLOC(done) )
return RET_ERROR_OOM;
done->finalfilename = donefilename(codename);
if( FAILEDTOALLOC(done->finalfilename) ) {
free(done);
return RET_ERROR_OOM;
fd = open(donefilename, O_RDONLY|O_NOCTTY|O_NOFOLLOW, 0666);
if( fd < 0 ) {
int e = errno;
if( e == EACCES || e == ENOENT ) {
free(donefilename);
return RET_OK;
}
fprintf(stderr, "Error opening file %s: %d=%s\n",
donefilename, e, strerror(e));
free(donefilename);
return RET_ERRNO(e);
}
len = 0;
while( len < 199 && (bytes=read(fd, buffer+len, 199-len)) > 0 ) {
len += bytes;
done->tempfilename = calc_addsuffix(done->finalfilename, "new");
if( FAILEDTOALLOC(done->tempfilename) ) {
free(done->finalfilename);
free(done);
return RET_ERROR_OOM;
}
if( bytes > 0 ) {
fprintf(stderr, "Unexpected long file %s!\n", donefilename);
(void)close(fd);
free(donefilename);
return RET_ERROR;
} else if( bytes < 0 ) {
done->file = fopen(done->tempfilename, "w+");
if( done->file == NULL ) {
int e = errno;
fprintf(stderr, "Error reading file %s: %d=%s\n",
donefilename, e, strerror(e));
(void)close(fd);
free(donefilename);
return RET_ERRNO(e);
fprintf(stderr, "Error %d creating '%s': %s\n",
e, done->tempfilename, strerror(e));
free(done->finalfilename);
free(done->tempfilename);
free(done);
return RET_ERROR;
}
free(donefilename);
(void)close(fd);
buffer[len] = '\0';
start = buffer;
/* for future extensibility: */
while( *start == ':' ) {
start++;
while( *start != ' ' && *start != '\0' )
start++;
if( *start == ' ' )
start++;
fprintf(done->file, "Updates already processed for %s:\n", codename);
*done_p = done;
return RET_OK;
}
void markdone_finish(struct markdonefile *done) {
bool error = false;
if( done == NULL )
return;
if( done->file == NULL )
error = true;
else {
if( ferror(done->file) != 0 ) {
fprintf(stderr, "An error occured writing to '%s'!\n",
done->tempfilename);
(void)fclose(done->file);
error = true;
} else if( fclose(done->file) != 0 ) {
int e = errno;
fprintf(stderr, "Error %d occured writing to '%s': %s!\n",
e, done->tempfilename, strerror(e));
error = true;
}
done->file = NULL;
}
if( checksums_matches(expected, cs_md5sum, start) )
return RET_NOTHING;
else
return RET_OK;
if( error )
(void)unlink(done->tempfilename);
else {
int i;
i = rename(done->tempfilename, done->finalfilename);
if( i != 0 ) {
int e = errno;
fprintf(stderr, "Error %d moving '%s' to '%s': %s!\n",
e, done->tempfilename,
done->finalfilename, strerror(e));
}
}
free(done->finalfilename);
free(done->tempfilename);
free(done);
}
void markdone_target(struct markdonefile *done, const char *identifier) {
fprintf(done->file, "Target %s\n", identifier);
}
void markdone_index(struct markdonefile *done, const char *file, const struct checksums *checksums) {
retvalue r;
size_t s;
const char *data;
r = checksums_getcombined(checksums, &data, &s);
if( !RET_IS_OK(r) )
return;
fprintf(done->file, "Index %s %s\n", file, data);
}
void markdone_cleaner(struct markdonefile *done) {
fprintf(done->file, "Delete\n");
}
retvalue donefile_create(const char *filename, const struct checksums *checksums) {
const char *md5sum;
const char *start;
size_t len;
ssize_t written;
int fd;
char *donefilename;
if( interrupted() )
return RET_ERROR_INTERRUPTED;
donefilename = calc_addsuffix(filename,"done");
if( donefilename == NULL )
/* the same for reading */
struct donefile {
char *filename;
char *linebuffer;
size_t linebuffer_size;
FILE *file;
};
retvalue donefile_open(const char *codename, struct donefile **done_p) {
struct donefile *done;
ssize_t s;
done = calloc(1, sizeof(struct donefile));
if( FAILEDTOALLOC(done) )
return RET_ERROR_OOM;
md5sum = checksums_getmd5sum(checksums);
fd = open(donefilename, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY|O_NOFOLLOW,
0666);
if( fd < 0 ) {
int e = errno;
fprintf(stderr, "Error creating file %s: %d=%s\n",
donefilename, e, strerror(e));
free(donefilename);
return RET_ERRNO(e);
done->filename = donefilename(codename);
if( FAILEDTOALLOC(done->filename) ) {
free(done);
return RET_ERROR_OOM;
}
start = md5sum;
len = strlen(md5sum);
written = 0;
while( len > 0 && (written=write(fd, start, len)) >= 0 ) {
start += written;
assert( len >= (size_t)written );
len -= written;
done->file = fopen(done->filename, "r");
if( done->file == NULL ) {
donefile_close(done);
return RET_NOTHING;
}
if( written < 0 ) {
int e = errno;
fprintf(stderr, "Error writing into %s: %d=%s\n",
donefilename, e, strerror(e));
free(donefilename);
(void)close(fd);
return RET_ERRNO(e);
s = getline(&done->linebuffer, &done->linebuffer_size, done->file);
if( s <= 0 || done->linebuffer[s-1] != '\n' ) {
/* if it cannot be read or is empty or not a text file,
* delete it, and do as if it never existed... */
unlink(done->filename);
donefile_close(done);
return RET_NOTHING;
}
if( close(fd) != 0 ) {
int e = errno;
fprintf(stderr, "Error writing %s: %d=%s\n",
donefilename, e, strerror(e));
free(donefilename);
return RET_ERRNO(e);
done->linebuffer[s-1] = '\0';
// TODO: check the first line?
*done_p = done;
return RET_OK;
}
void donefile_close(struct donefile *done) {
if( done == NULL )
return;
// TODO: check return, only print a warning, though,
// no need to interrupt anything.
if( done->file != NULL )
fclose(done->file);
free(done->linebuffer);
free(done->filename);
free(done);
}
retvalue donefile_nexttarget(struct donefile *done, const char **identifier_p) {
ssize_t s;
while( strncmp(done->linebuffer, "Target ", 7) != 0 ) {
s = getline(&done->linebuffer, &done->linebuffer_size, done->file);
if( s <= 0 || done->linebuffer[s-1] != '\n' )
/* Malformed line, ignore the rest... */
return RET_NOTHING;
done->linebuffer[s-1] = '\0';
}
free(donefilename);
/* do not process a second time */
done->linebuffer[0] = '\0';
/* and return the identifier part */
*identifier_p = done->linebuffer + 7;
return RET_OK;
}
void donefile_delete(const char *filename) {
char *donefilename = calc_addsuffix(filename,"done");
unlink(donefilename);
free(donefilename);
bool donefile_nextindex(struct donefile *done, const char **filename_p, struct checksums **checksums_p) {
char *p;
ssize_t s;
retvalue r;
s = getline(&done->linebuffer, &done->linebuffer_size, done->file);
if( s <= 0 || done->linebuffer[s-1] != '\n' ) {
done->linebuffer[0] = '\0';
return false;
}
done->linebuffer[s-1] = '\0';
if( strncmp(done->linebuffer, "Index ", 6) != 0 )
return false;
p = done->linebuffer + 6;
*filename_p = p;
p = strchr(p, ' ');
if( p == NULL )
return false;
*(p++) = '\0';
r = checksums_parse(checksums_p, p);
return RET_IS_OK(r);
}
bool donefile_iscleaner(struct donefile *done) {
ssize_t s;
s = getline(&done->linebuffer, &done->linebuffer_size, done->file);
if( s <= 0 || done->linebuffer[s-1] != '\n' ) {
done->linebuffer[0] = '\0';
return false;
}
done->linebuffer[s-1] = '\0';
return strcmp(done->linebuffer, "Delete") == 0;
}
......@@ -7,7 +7,19 @@
struct checksums;
retvalue donefile_isold(const char *filename, const struct checksums *expected);
retvalue donefile_create(const char *filename, const struct checksums *);
void donefile_delete(const char *filename);
struct markdonefile;
retvalue markdone_create(const char *, /*@out@*/struct markdonefile **);
void markdone_finish(/*@only@*/struct markdonefile *);
void markdone_target(struct markdonefile *, const char *);
void markdone_index(struct markdonefile *, const char *, const struct checksums *);
void markdone_cleaner(struct markdonefile *);
struct donefile;
retvalue donefile_open(const char *, /*@out@*/struct donefile **);
void donefile_close(/*@only@*/struct donefile *);
retvalue donefile_nexttarget(struct donefile *, /*@out@*/const char **);
bool donefile_nextindex(struct donefile *, /*@out@*/const char **, /*@out@*/struct checksums **);
bool donefile_iscleaner(struct donefile *);
#endif
......@@ -61,6 +61,7 @@
#include "override.h"
#include "log.h"
#include "copypackages.h"
#include "uncompression.h"
#ifndef STD_BASE_DIR
#define STD_BASE_DIR "."
......@@ -3217,6 +3218,9 @@ int main(int argc,char *argv[]) {
global.logdir = x_logdir;
global.methoddir = x_methoddir;
global.listdir = x_listdir;
uncompressions_check();
a = all_actions;
while( a->name != NULL ) {
if( strcasecmp(a->name,argv[optind]) == 0 ) {
......
This diff is collapsed.
#ifndef REPREPRO_REMOTEREPOSITORY_H
#define REPREPRO_REMOTEREPOSITORY_H
#ifndef REPREPRO_ERROR_H
#include "error.h"
#warning "What's hapening here?"
#endif
#ifndef REPREPRO_APTMETHOD_H
#include "aptmethod.h"
#endif
#ifndef REPREPRO_DONEFILE_H
#include "donefile.h"
#endif
struct remote_repository;
struct remote_distribution;
struct remote_index;
/* register repository, strings as stored by reference */
struct remote_repository *remote_repository_prepare(const char *name, const char *method, const char *fallback, const struct strlist *config);
/* register remote distribution of the given repository */
struct remote_distribution *remote_distribution_prepare(struct remote_repository *, const char *suite, bool ignorerelease, const char *verifyrelease, bool flat, bool *ignorehashes);
void remote_repository_free(/*@only@*/struct remote_repository *);
/* create aptmethods for all of yet created repositories */
retvalue remote_startup(struct aptmethodrun *);
retvalue remote_preparemetalists(struct aptmethodrun *, bool nodownload);
retvalue remote_preparelists(struct aptmethodrun *, bool nodownload);
struct remote_index *remote_index(struct remote_distribution *, const char *architecture, const char *component, const char *packagetype);
struct remote_index *remote_flat_index(struct remote_distribution *, const char *packagetype);
/* returns the name of the prepared uncompressed file */
/*@observer@*/const char *remote_index_file(const struct remote_index*);
/*@observer@*/const char *remote_index_basefile(const struct remote_index*);
/*@observer@*/struct aptmethod *remote_aptmethod(const struct remote_distribution *);
bool remote_index_isnew(const struct remote_index *, struct donefile *);
void remote_index_needed(struct remote_index *);
void remote_index_markdone(const struct remote_index *, struct markdonefile *);
char *genlistsfilename(/*@null@*/const char * /*type*/, unsigned int /*count*/, ...) __attribute__((sentinel));
#endif
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -31,11 +31,8 @@ void updates_freeupdatedistributions(/*@only@*/struct update_distribution *d);
retvalue updates_calcindices(struct update_pattern *, struct distribution *, bool fast, /*@out@*/struct update_distribution **);
/* remove all files ${listdir}/${distribution}_* that will not be needed. */
retvalue updates_clearlists(struct update_distribution *distributions);
retvalue updates_update(struct database *, struct update_distribution *distributions, bool nolistsdownload, bool skipold, struct strlist *dereferencedfilekeys, enum spacecheckmode mode, off_t reserveddb, off_t reservedother);
retvalue updates_checkupdate(struct database *, struct update_distribution *distributions, bool nolistsdownload, bool skipold);
retvalue updates_predelete(struct database *, struct update_distribution *distributions, bool nolistsdownload, bool skipold, struct strlist *dereferencedfilekeys);
retvalue updates_update(struct database *, struct update_distribution *, bool nolistsdownload, bool skipold, struct strlist *dereferencedfilekeys, enum spacecheckmode mode, off_t reserveddb, off_t reservedother);
retvalue updates_checkupdate(struct database *, struct update_distribution *, bool nolistsdownload, bool skipold);
retvalue updates_predelete(struct database *, struct update_distribution *, bool nolistsdownload, bool skipold, struct strlist *dereferencedfilekeys);
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment