Commit f8954504 authored by Bernhard Link's avatar Bernhard Link

add code to choose which index files to download

parent 446c9ffc
......@@ -113,6 +113,9 @@ struct remote_index {
struct remote_distribution *from;
/* what to download? .gz better than .bz2? and so on */
struct encoding_preferences downloadas;
/* remote filename as to be found in Release file*/
char *filename_in_release;
......@@ -802,6 +805,131 @@ static inline void remote_index_oldfiles(struct remote_index *ri, /*@null@*/stru
}
}
static inline enum compression firstsupportedencoding(const struct encoding_preferences *downloadas) {
int e;
if( downloadas->count == 0 )
/* if nothing is specified, get .gz */
return c_gzip;
for( e = 0 ; e < downloadas->count ; e++ ) {
enum compression c = downloadas->requested[e];
if( uncompression_supported(c) )
return c;
}
// TODO: instead give an good warning or error...
return c_gzip;
}
static inline retvalue find_requested_encoding(struct remote_index *ri, const char *releasefile) {
int e;
enum compression c,
/* the most-preferred requested but unsupported */
unsupported = c_COUNT,
/* the best unrequested but supported */
unrequested = c_COUNT;
if( ri->downloadas.count > 0 ) {
bool found = false;
for( e = 0 ; e < ri->downloadas.count ; e++ ) {
c = ri->downloadas.requested[e];
if( ri->ofs[c] < 0 )
continue;
if( uncompression_supported(c) ) {
ri->compression = c;
return RET_OK;
} else if( unsupported == c_COUNT )
unsupported = c;
}
/* nothing that is both requested by the user and supported
* and listed in the Release file found, check what is there
* to get a meaningfull error message */
for( c = 0 ; c < c_COUNT ; c++ ) {
if( ri->ofs[c] < 0 )
continue;
found = true;
if( uncompression_supported(c) )
unrequested = c;
}
if( !found ) {
// TODO: might be nice to check for not-yet-even
// known about compressions and say they are not
// yet know yet instead then here...
fprintf(stderr,
"Could not find '%s' within '%s'\n",
ri->filename_in_release, releasefile);
return RET_ERROR_WRONG_MD5;
}
if( !found ) {
fprintf(stderr,
"Error: '%s' only lists unusable or unrequested compressions of '%s'.\n"
"Try e.g the '%s' option (or check what it is set to) to make more useable.\n"
"Or change your DownloadListsAs to request e.g. '%s'.\n",
releasefile, ri->filename_in_release,
uncompression_option[unsupported],
uncompression_suffix[unrequested]);
return RET_ERROR;
}
if( unsupported != c_COUNT ) {
fprintf(stderr,
"Error: '%s' only lists unusable compressions of '%s'.\n"
"Try e.g the '%s' option (or check what it is set to) to make more useable.\n",
releasefile, ri->filename_in_release,
uncompression_option[unsupported]);
return RET_ERROR;
}
if( unrequested != c_COUNT ) {
fprintf(stderr,
"Error: '%s' only lists unrequested compressions of '%s'.\n"
"Try changing your DownloadListsAs to request e.g. '%s'.\n",
releasefile, ri->filename_in_release,
uncompression_suffix[unrequested]);
return RET_ERROR;
}
fprintf(stderr,
"Error: '%s' lists no requested and usable compressions of '%s'.\n",
releasefile, ri->filename_in_release);
return RET_ERROR;
}
/* When nothing specified, use the newest compression.
* This might make it slow on older computers (and perhaps
* on relatively new ones, too), but usually bandwidth costs
* and your time not.
* And you can always configure it to prefer a faster one...
*/
ri->compression = c_COUNT;
for( c = 0 ; c < c_COUNT ; c++ ) {
if( ri->ofs[c] < 0 )
continue;
if( uncompression_supported(c) )
ri->compression = c;
else
unsupported = c;
}
if( ri->compression == c_COUNT ) {
if( unsupported != c_COUNT ) {
fprintf(stderr,
"Error: '%s' only lists unusable compressions of '%s'.\n"
"Try e.g the '%s' option (or check what it is set to) to enable more.\n",
releasefile, ri->filename_in_release,
uncompression_option[unsupported]);
return RET_ERROR;
}
fprintf(stderr,
"Could not find '%s' within '%s'\n",
ri->filename_in_release, releasefile);
return RET_ERROR_WRONG_MD5;
}
return RET_OK;
}
static inline retvalue queueindex(struct remote_distribution *rd, struct remote_index *ri, bool nodownload, /*@null@*/struct cachedlistfile *oldfiles, bool *tobecontinued) {
struct remote_repository *rr = rd->repository;
enum compression c;
......@@ -816,12 +944,26 @@ static inline retvalue queueindex(struct remote_distribution *rd, struct remote_
if( nodownload )
return RET_OK;
/* we do not know what upstream uses, just assume .gz */
toget = calc_addsuffix(ri->filename_in_release, "gz");
ri->compression = c_gzip;
/* Without a Release file there is no knowing what compression
* upstream uses. Situation gets worse as we miss the means yet
* to try something and continue with something else if it
* fails (as aptmethod error reporting would need to be extended
* for it). Thus use the first supported of the requested and
* use .gz as fallback.
*
* Using this preferences means users will have to set it for
* IgnoreRelease-distributions, that default in an parent
* rule to something already, but better than to invent another
* mechanism to specify this... */
c = firstsupportedencoding(&ri->downloadas);
assert( uncompression_supported(c) );
ri->compression = c;
toget = mprintf("%s%s", ri->filename_in_release,
uncompression_suffix[c]);
r = aptmethod_queueindexfile(rr->download, rd->suite_base_dir,
toget, ri->cachefilename, NULL, c_gzip, NULL);
toget, ri->cachefilename, NULL, c, NULL);
free(toget);
return r;
}
......@@ -921,23 +1063,16 @@ static inline retvalue queueindex(struct remote_distribution *rd, struct remote_
return RET_ERROR_MISSING;
}
/* assume the more newer the compression the better
* (though on low end architectures
* the opposite holds, so TODO: make this configurable */
r = find_requested_encoding(ri, rd->releasefile);
assert( r != RET_NOTHING );
if( RET_WAS_ERROR(r) )
return r;
ri->compression = c_COUNT;
for( c = 0 ; c < c_COUNT ; c++ ) {
if( ri->ofs[c] >= 0 && uncompression_supported(c) )
ri->compression = c;
}
if( ri->compression == c_COUNT ) {
fprintf(stderr,
"Could not find '%s' within '%s'\n",
ri->filename_in_release, rd->releasefile);
return RET_ERROR_WRONG_MD5;
assert( ri->compression < c_COUNT );
assert( uncompression_supported(ri->compression) );
}
ofs = ri->ofs[ri->compression];
assert( ofs >= 0 );
/* as those checksums might be overwritten with completed data,
* this assumes that the uncompressed checksums for one index is never
......@@ -1015,7 +1150,7 @@ retvalue remote_preparelists(struct aptmethodrun *run, bool nodownload) {
return RET_OK;
}
static struct remote_index *addindex(struct remote_distribution *rd, /*@only@*/char *cachefilename, /*@only@*/char *filename) {
static struct remote_index *addindex(struct remote_distribution *rd, /*@only@*/char *cachefilename, /*@only@*/char *filename, /*@null@*/const struct encoding_preferences *downloadas) {
struct remote_index *ri, **last;
enum compression c;
const char *cachebasename;
......@@ -1043,13 +1178,18 @@ static struct remote_index *addindex(struct remote_distribution *rd, /*@only@*/c
ri->cachefilename = cachefilename;
ri->cachebasename = cachebasename;
ri->filename_in_release = filename;
// TODO: perhaps try to calculate some form of intersections
// instead of just using the shorter one...
if( downloadas != NULL && (ri->downloadas.count == 0
|| ri->downloadas.count > downloadas->count) )
ri->downloadas = *downloadas;
for( c = 0 ; c < c_COUNT ; c++ )
ri->ofs[c] = -1;
ri->diff_ofs = -1;
return ri;
}
struct remote_index *remote_index(struct remote_distribution *rd, const char *architecture, const char *component, packagetype_t packagetype) {
struct remote_index *remote_index(struct remote_distribution *rd, const char *architecture, const char *component, packagetype_t packagetype, /*@null@*/const struct encoding_preferences *downloadas) {
char *cachefilename, *filename_in_release;
assert( !rd->flat );
......@@ -1075,7 +1215,7 @@ struct remote_index *remote_index(struct remote_distribution *rd, const char *ar
} else {
assert( "Unexpected package type" == NULL );
}
return addindex(rd, cachefilename, filename_in_release);
return addindex(rd, cachefilename, filename_in_release, downloadas);
}
void cachedlistfile_need_index(struct cachedlistfile *list, const char *repository, const char *suite, const char *architecture, const char *component, packagetype_t packagetype) {
......@@ -1094,7 +1234,7 @@ void cachedlistfile_need_index(struct cachedlistfile *list, const char *reposito
}
}
struct remote_index *remote_flat_index(struct remote_distribution *rd, packagetype_t packagetype) {
struct remote_index *remote_flat_index(struct remote_distribution *rd, packagetype_t packagetype, /*@null@*/const struct encoding_preferences *downloadas) {
char *cachefilename, *filename_in_release;
assert( rd->flat );
......@@ -1111,7 +1251,7 @@ struct remote_index *remote_flat_index(struct remote_distribution *rd, packagety
} else {
assert( "Unexpected package type" == NULL );
}
return addindex(rd, cachefilename, filename_in_release);
return addindex(rd, cachefilename, filename_in_release, downloadas);
}
void cachedlistfile_need_flat_index(struct cachedlistfile *list, const char *repository, const char *suite, packagetype_t packagetype) {
......
......@@ -33,8 +33,16 @@ retvalue remote_startup(struct aptmethodrun *);
retvalue remote_preparemetalists(struct aptmethodrun *, bool nodownload);
retvalue remote_preparelists(struct aptmethodrun *, bool nodownload);
struct remote_index *remote_index(struct remote_distribution *, const char *architecture, const char *component, packagetype_t);
struct remote_index *remote_flat_index(struct remote_distribution *, packagetype_t);
struct encoding_preferences {
/* number of preferences, 0 means use default */
unsigned short count;
/* a list of compressions to use,
* a c_COUNT means end of the list... */
enum compression requested[c_COUNT];
};
struct remote_index *remote_index(struct remote_distribution *, const char *architecture, const char *component, packagetype_t, const struct encoding_preferences *);
struct remote_index *remote_flat_index(struct remote_distribution *, packagetype_t, const struct encoding_preferences *);
/* returns the name of the prepared uncompressed file */
/*@observer@*/const char *remote_index_file(const struct remote_index*);
......
......@@ -41,6 +41,10 @@
const char * const uncompression_suffix[c_COUNT] = {
"", ".gz", ".bz2", ".lzma" };
/* So help messages can hint what option to try */
const char * const uncompression_option[c_COUNT] = {
NULL, NULL, "--bunzip2", "--unlzma" };
/*@null@*/ char *extern_uncompressors[c_COUNT] = { NULL, NULL, NULL, NULL};
......
......@@ -4,6 +4,8 @@
/* "", ".gz", ... */
extern const char * const uncompression_suffix[c_COUNT];
extern /*@null@*/ char *extern_uncompressors[c_COUNT];
/* so help messages know which option to cite: */
extern const char * const uncompression_option[c_COUNT];
/* there are two different modes: uncompress a file to memory,
* or uncompress (possibly multiple files) on the filesystem,
......
......@@ -163,6 +163,8 @@ struct update_pattern {
component_t flat;
//e.g. "IgnoreRelease: Yes" for 1 (default is 0)
bool ignorerelease;
/* the form in which index files are preferably downloaded */
struct encoding_preferences downloadlistsas;
/* if the specific field is there (to destinguish from an empty one) */
bool ignorehashes_set;
bool ignorerelease_set;
......@@ -171,6 +173,7 @@ struct update_pattern {
bool udebcomponents_set;
bool includecondition_set;
bool config_set;
bool downloadlistsas_set;
/* to check circular references */
bool visited;
......@@ -852,15 +855,21 @@ static retvalue getorigins(struct update_distribution *d) {
static inline bool addremoteindex(struct update_origin *origin, struct target *target, struct update_target *updatetargets, const char *architecture, const char *component) {
struct update_index_connector *uindex;
const struct update_pattern *p;
uindex = calloc(1, sizeof(struct update_index_connector));
if( FAILEDTOALLOC(uindex) )
return false;
p = origin->pattern;
while( p != NULL && !p->downloadlistsas_set )
p = p->pattern_from;
uindex->origin = origin;
uindex->remote = remote_index(origin->from,
architecture, component,
target->packagetype_atom);
target->packagetype_atom,
(p == NULL)?NULL:&p->downloadlistsas);
if( FAILEDTOALLOC(uindex->remote) ) {
free(uindex);
return false;
......@@ -953,12 +962,22 @@ static retvalue addorigintotarget(struct update_origin *origin, struct target *t
static retvalue addflatorigintotarget(struct update_origin *origin, struct target *target, struct update_target *updatetargets ) {
const struct update_pattern *p;
const struct strlist *a_from, *a_into;
const struct encoding_preferences *downloadlistsas;
int ai;
assert( origin != NULL );
if( target->packagetype_atom == pt_udeb )
return RET_NOTHING;
p = origin->pattern;
while( p != NULL && !p->downloadlistsas_set )
p = p->pattern_from;
if( p == NULL )
downloadlistsas = NULL;
else
downloadlistsas = &p->downloadlistsas;
p = origin->pattern;
while( p != NULL && !atom_defined(p->flat) )
p = p->pattern_from;
......@@ -976,9 +995,11 @@ static retvalue addflatorigintotarget(struct update_origin *origin, struct targe
if( uindex == NULL )
return RET_ERROR_OOM;
uindex->origin = origin;
uindex->remote = remote_flat_index(origin->from,
target->packagetype_atom);
target->packagetype_atom,
downloadlistsas);
if( FAILEDTOALLOC(uindex->remote) ) {
free(uindex);
return RET_ERROR_OOM;
......@@ -1003,7 +1024,8 @@ static retvalue addflatorigintotarget(struct update_origin *origin, struct targe
uindex->origin = origin;
uindex->remote = remote_flat_index(origin->from,
target->packagetype_atom);
target->packagetype_atom,
downloadlistsas);
if( FAILEDTOALLOC(uindex->remote) ) {
free(uindex);
return RET_ERROR_OOM;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment