Commit f209a893 authored by Bernhard Link's avatar Bernhard Link

add --onlysmalldeletes

parent c47737ca
2010-05-07
* add --onlysmalldeletes option that cancels pulls
and updates that delete more than 20% of some target
(but at least 10 packages). The change also causes
update no longer claiming to get packages if there are
not any packages to get...
2010-04-30
* change parsing of .changes lines to cope with
N_V.orig-X.tar.C files where V.orig-X does not survive
......
......@@ -875,7 +875,7 @@ retvalue distribution_exportlist(enum exportwhen when, struct distribution *dist
}
for( d=distributions; d != NULL; d = d->next ) {
if( !d->selected )
if( d->omitted || !d->selected )
continue;
if( d->lookedat && (RET_IS_OK(d->status) ||
( d->status == RET_NOTHING && when != EXPORT_CHANGED) ||
......@@ -889,7 +889,7 @@ retvalue distribution_exportlist(enum exportwhen when, struct distribution *dist
result = RET_NOTHING;
for( d=distributions; d != NULL; d = d->next ) {
if( !d->selected )
if( d->omitted || !d->selected )
continue;
if( !d->lookedat ) {
if( verbose >= 30 )
......
......@@ -102,6 +102,8 @@ struct distribution {
bool readonly;
/* tracking information might be obsolete */
bool needretrack;
/* omitted because of --onlysmalldeletes */
bool omitted;
};
retvalue distribution_get(struct distribution *all, const char *name, bool lookedat, /*@out@*/struct distribution **);
......
......@@ -427,6 +427,16 @@ and might change meaning without any further notice.
When downloading packages, show each completed percent of completed package
downloads together with the size of completely downloaded packages.
(Repeating this option increases the frequency of this output).
.TP
.B \-\-onlysmalldeletes
The pull and update commands will skip every distribution in which one
target loses more than 20% of its packages (and at least 10).
Using this option (or putting it in the options config file) can
avoid removing large quantities of data but means you might often
give
.B \-\-noonlysmalldeletes
to override it.
.SH COMMANDS
.TP
.BR export " [ " \fIcodenames\fP " ]"
......
......@@ -79,6 +79,7 @@ extern struct global_config {
/* flags: */
bool keepdirectories;
bool keeptemporaries;
bool onlysmalldeletes;
/* verbosity of downloading statistics */
int showdownloadpercent;
} global;
......
......@@ -134,7 +134,7 @@ static off_t reservedotherspace = 1024*1024;
* to change something owned by lower owners. */
enum config_option_owner config_state,
#define O(x) owner_ ## x = CONFIG_OWNER_DEFAULT
O(fast), O(x_morguedir), O(x_outdir), O(x_basedir), O(x_distdir), O(x_dbdir), O(x_listdir), O(x_confdir), O(x_logdir), O(x_methoddir), O(x_section), O(x_priority), O(x_component), O(x_architecture), O(x_packagetype), O(nothingiserror), O(nolistsdownload), O(keepunusednew), O(keepunreferenced), O(keeptemporaries), O(keepdirectories), O(askforpassphrase), O(skipold), O(export), O(waitforlock), O(spacecheckmode), O(reserveddbspace), O(reservedotherspace), O(guessgpgtty), O(verbosedatabase), O(gunzip), O(bunzip2), O(unlzma), O(unxz), O(gnupghome), O(listformat), O(listmax), O(listskip);
O(fast), O(x_morguedir), O(x_outdir), O(x_basedir), O(x_distdir), O(x_dbdir), O(x_listdir), O(x_confdir), O(x_logdir), O(x_methoddir), O(x_section), O(x_priority), O(x_component), O(x_architecture), O(x_packagetype), O(nothingiserror), O(nolistsdownload), O(keepunusednew), O(keepunreferenced), O(keeptemporaries), O(keepdirectories), O(askforpassphrase), O(skipold), O(export), O(waitforlock), O(spacecheckmode), O(reserveddbspace), O(reservedotherspace), O(guessgpgtty), O(verbosedatabase), O(gunzip), O(bunzip2), O(unlzma), O(unxz), O(gnupghome), O(listformat), O(listmax), O(listskip), O(onlysmalldeletes);
#undef O
#define CONFIGSET(variable,value) if(owner_ ## variable <= config_state) { \
......@@ -3612,6 +3612,7 @@ LO_KEEPUNNEEDEDLISTS,
LO_NOTHINGISERROR,
LO_NOLISTDOWNLOAD,
LO_ASKPASSPHRASE,
LO_ONLYSMALLDELETES,
LO_KEEPDIRECTORIES,
LO_KEEPTEMPORARIES,
LO_FAST,
......@@ -3624,6 +3625,7 @@ LO_NOKEEPUNNEEDEDLISTS,
LO_NONOTHINGISERROR,
LO_LISTDOWNLOAD,
LO_NOASKPASSPHRASE,
LO_NOONLYSMALLDELETES,
LO_NOKEEPDIRECTORIES,
LO_NOKEEPTEMPORARIES,
LO_NOFAST,
......@@ -3802,6 +3804,12 @@ static void handle_option(int c, const char *argument) {
case LO_NOKEEPTEMPORARIES:
CONFIGGSET(keeptemporaries, false);
break;
case LO_ONLYSMALLDELETES:
CONFIGGSET(onlysmalldeletes, true);
break;
case LO_NOONLYSMALLDELETES:
CONFIGGSET(onlysmalldeletes, false);
break;
case LO_KEEPDIRECTORIES:
CONFIGGSET(keepdirectories, true);
break;
......@@ -4138,6 +4146,7 @@ int main(int argc,char *argv[]) {
{"keepunreferencedfiles", no_argument, &longoption, LO_KEEPUNREFERENCED},
{"keepunusednewfiles", no_argument, &longoption, LO_KEEPUNUSEDNEW},
{"keepunneededlists", no_argument, &longoption, LO_KEEPUNNEEDEDLISTS},
{"onlysmalldeletes", no_argument, &longoption, LO_ONLYSMALLDELETES},
{"keepdirectories", no_argument, &longoption, LO_KEEPDIRECTORIES},
{"keeptemporaries", no_argument, &longoption, LO_KEEPTEMPORARIES},
{"ask-passphrase", no_argument, &longoption, LO_ASKPASSPHRASE},
......@@ -4147,6 +4156,7 @@ int main(int argc,char *argv[]) {
{"nokeepunreferencedfiles", no_argument, &longoption, LO_NOKEEPUNREFERENCED},
{"nokeepunusednewfiles", no_argument, &longoption, LO_NOKEEPUNUSEDNEW},
{"nokeepunneededlists", no_argument, &longoption, LO_NOKEEPUNNEEDEDLISTS},
{"noonlysmalldeletes", no_argument, &longoption, LO_NOONLYSMALLDELETES},
{"nokeepdirectories", no_argument, &longoption, LO_NOKEEPDIRECTORIES},
{"nokeeptemporaries", no_argument, &longoption, LO_NOKEEPTEMPORARIES},
{"noask-passphrase", no_argument, &longoption, LO_NOASKPASSPHRASE},
......
......@@ -750,6 +750,25 @@ static retvalue pull_search(/*@null@*/FILE *out,struct database *database,struct
return result;
}
static bool pull_isbigdelete(struct pull_distribution *d) {
struct pull_target *u, *v;
for( u = d->targets ; u != NULL ; u=u->next ) {
if( u->ignoredelete )
continue;
if( upgradelist_isbigdelete(u->upgradelist) ) {
d->distribution->omitted = true;
for( v = d->targets ; v != NULL ; v = v->next ) {
upgradelist_free(v->upgradelist);
v->upgradelist = NULL;
}
return true;
}
}
return false;
}
static void pull_from_callback(void *privdata, const char **rule_p, const char **from_p) {
struct pull_source *source = privdata;
......@@ -926,6 +945,14 @@ retvalue pull_update(struct database *database, struct pull_distribution *distri
printf("Installing (and possibly deleting) packages...\n");
for( d=distributions ; d != NULL ; d=d->next) {
if( global.onlysmalldeletes ) {
if( pull_isbigdelete(d) ) {
fprintf(stderr,
"Not processing '%s' because of --onlysmalldeletes\n",
d->distribution->codename);
continue;
}
}
r = pull_install(database, d);
RET_UPDATE(result,r);
if( RET_WAS_ERROR(r) )
......
......@@ -26,6 +26,7 @@ runtest copy
runtest buildneeding
runtest morgue
runtest diffgeneration
runtest onlysmalldeletes
set +v +x
echo
......
......@@ -164,9 +164,6 @@ stdout
-v4*= nothing to do for 'u|1234|a|x'
-v3*= processing updates for '1234|a|x'
#-v5*= reading './lists/flattest_flatsource_Packages'
-v0*=Getting packages...
-v1*=Shutting down aptmethods...
-v0*=Installing (and possibly deleting) packages...
EOF
cat > flatsource/Packages <<EOF
......@@ -200,9 +197,6 @@ stdout
-v4*= nothing to do for 'u|1234|a|x'
-v3*= processing updates for '1234|a|x'
#-v5*= reading './lists/flattest_flatsource_Packages'
-v0*=Getting packages...
-v1*=Shutting down aptmethods...
-v0*=Installing (and possibly deleting) packages...
EOF
cat > flatsource/Packages <<EOF
......
......@@ -366,12 +366,6 @@ if [ $verbosity -ge 5 ] ; then
echo " reading './lists/base_codename2_a_x_Packages'" >>results.expected
echo " marking everything to be deleted" >>results.expected
echo " reading './lists/base_codename1_a_x_Packages'" >>results.expected ; fi
if [ $verbosity -ge 0 ] ; then
echo "Getting packages..." >>results.expected ; fi
if [ $verbosity -ge 1 ] ; then
echo "Shutting down aptmethods..." >>results.expected ; fi
if [ $verbosity -ge 0 ] ; then
echo "Installing (and possibly deleting) packages..." >>results.expected ; fi
dodiff results.expected results
mv results.expected results2.expected
......@@ -482,12 +476,6 @@ echo " marking everything to be deleted" >>results.expected
echo " reading './lists/base_codename2_a_x_Packages'" >>results.expected
echo " reading './lists/base_codename1_a_x_Packages'" >>results.expected
fi
if [ $verbosity -ge 0 ] ; then
echo "Getting packages..." >>results.expected ; fi
if [ $verbosity -ge 1 ] ; then
echo "Shutting down aptmethods..." >>results.expected ; fi
if [ $verbosity -ge 0 ] ; then
echo "Installing (and possibly deleting) packages..." >>results.expected ; fi
dodiff results.expected results
testrun - -b . update codename2 codename1 3<<EOF
......
......@@ -640,9 +640,6 @@ stdout
-v3*= processing updates for 'boring|main|${FAKEARCHITECTURE}'
#-v5*= reading './lists/b_x_Packages'
-v5*= reading './lists/b_x_main_${FAKEARCHITECTURE}_Packages'
-v0*=Getting packages...
-v1*=Shutting down aptmethods...
-v0*=Installing (and possibly deleting) packages...
stderr
EOF
......
#!/bin/bash
set -e
if [ "x$TESTINCSETUP" != "xissetup" ] ; then
source $(dirname $0)/test.inc
fi
mkdir conf
cat >conf/distributions <<EOF
Codename: test
Architectures: $FAKEARCHITECTURE source
Components: all
Codename: copy
Architectures: $FAKEARCHITECTURE source
Components: all
Pull: rule
EOF
touch conf/updates
cat >conf/pulls <<EOF
Name: rule
From: test
EOF
cat >conf/incoming <<EOF
Name: i
Tempdir: tmp
Incomingdir: i
Default: test
EOF
cat >conf/options <<EOF
onlysmalldeletes
EOF
mkdir i
cd i
for i in $(seq 1 40) ; do
PACKAGE=a$i EPOCH="" VERSION=$i REVISION="" SECTION="many" genpackage.sh
mv test.changes a$i.changes
done
cd ..
cat > pi.rules <<EOF
=Data seems not to be signed trying to use directly...
stdout
-v2*=Created directory "./db"
-v2*=Created directory "./tmp"
-v2*=Created directory "./pool"
-v2*=Created directory "./pool/all"
-v2*=Created directory "./pool/all/a"
-v0*=Exporting indices...
-v2*=Created directory "./dists"
-v2*=Created directory "./dists/test"
-v2*=Created directory "./dists/test/all"
-v2*=Created directory "./dists/test/all/binary-${FAKEARCHITECTURE}"
-v6*= looking for changes in 'test|all|${FAKEARCHITECTURE}'...
-v6*= creating './dists/test/all/binary-${FAKEARCHITECTURE}/Packages' (uncompressed,gzipped)
-v2*=Created directory "./dists/test/all/source"
-v6*= looking for changes in 'test|all|source'...
-v6*= creating './dists/test/all/source/Sources' (gzipped)
EOF
for i in $(seq 1 40) ; do
cat >>pi.rules <<EOF
-v2*=Created directory "./pool/all/a/a$i"
-d1*=db: 'pool/all/a/a${i}/a${i}_${i}.dsc' added to checksums.db(pool).
-d1*=db: 'pool/all/a/a${i}/a${i}_${i}.tar.gz' added to checksums.db(pool).
-d1*=db: 'pool/all/a/a${i}/a${i}_${i}_${FAKEARCHITECTURE}.deb' added to checksums.db(pool).
-d1*=db: 'pool/all/a/a${i}/a${i}-addons_${i}_all.deb' added to checksums.db(pool).
-d1*=db: 'a${i}' added to packages.db(test|all|source).
-d1*=db: 'a${i}' added to packages.db(test|all|${FAKEARCHITECTURE}).
-d1*=db: 'a${i}-addons' added to packages.db(test|all|${FAKEARCHITECTURE}).
-v1*=deleting './i/a${i}.changes'...
-v1*=deleting './i/a${i}_${i}.dsc'...
-v1*=deleting './i/a${i}_${i}.tar.gz'...
-v1*=deleting './i/a${i}_${i}_${FAKEARCHITECTURE}.deb'...
-v1*=deleting './i/a${i}-addons_${i}_all.deb'...
EOF
done
testrun pi -b . processincoming i
dodo rmdir i
rm pi.rules
cat >pull.rules <<EOF
stdout
-v0*=Calculating packages to pull...
-v3*= pulling into 'copy|all|source'
-v5*= looking what to get from 'test|all|source'
-v3*= pulling into 'copy|all|${FAKEARCHITECTURE}'
-v5*= looking what to get from 'test|all|${FAKEARCHITECTURE}'
-v0*=Installing (and possibly deleting) packages...
-v0*=Exporting indices...
-v2*=Created directory "./dists/copy"
-v2*=Created directory "./dists/copy/all"
-v2*=Created directory "./dists/copy/all/binary-${FAKEARCHITECTURE}"
-v6*= looking for changes in 'copy|all|${FAKEARCHITECTURE}'...
-v6*= creating './dists/copy/all/binary-${FAKEARCHITECTURE}/Packages' (uncompressed,gzipped)
-v2*=Created directory "./dists/copy/all/source"
-v6*= looking for changes in 'copy|all|source'...
-v6*= creating './dists/copy/all/source/Sources' (gzipped)
EOF
for i in $(seq 1 40) ; do
cat >>pull.rules <<EOF
-d1*=db: 'a${i}' added to packages.db(copy|all|source).
-d1*=db: 'a${i}' added to packages.db(copy|all|${FAKEARCHITECTURE}).
-d1*=db: 'a${i}-addons' added to packages.db(copy|all|${FAKEARCHITECTURE}).
EOF
done
testrun pull -b . pull
rm pull.rules
sed -e 's/Pull: rule/Pull: -/' -i conf/distributions
testrun - -b . pull 3<<EOF
stdout
-v0*=Calculating packages to pull...
-v3*= pulling into 'copy|all|source'
-v5*= marking everything to be deleted
-v3*= pulling into 'copy|all|${FAKEARCHITECTURE}'
#-v5*= marking everything to be deleted
-v0*=Installing (and possibly deleting) packages...
stderr
*=Not processing 'copy' because of --onlysmalldeletes
EOF
sed -e 's/Pull: -/Update: -/' -i conf/distributions
testrun - -b . --noskipold update 3<<EOF
stdout
-v2*=Created directory "./lists"
-v0*=Calculating packages to get...
-v3*= processing updates for 'copy|all|source'
-v5*= marking everything to be deleted
-v3*= processing updates for 'copy|all|${FAKEARCHITECTURE}'
#-v5*= marking everything to be deleted
stderr
*=Not processing updates for 'copy' because of --onlysmalldeletes!
EOF
rm -r conf
rm -r db
rm -r pool
rm -r dists
rmdir tmp
rmdir lists
testsuccess
......@@ -253,9 +253,6 @@ stdout
-v0*=Calculating packages to get...
-v3*= processing updates for 'test|main|coal'
-v5*= reading './lists/fromsource_sourcedistribution_main_coal_Packages'
-v0*=Getting packages...
-v1*=Shutting down aptmethods...
-v0*=Installing (and possibly deleting) packages...
EOF
dodiff dists/sourcedistribution/main/binary-coal/Packages lists/fromsource_sourcedistribution_main_coal_Packages
......@@ -279,9 +276,6 @@ stdout
-v0*=Calculating packages to get...
-v3*= processing updates for 'test|main|coal'
-v5*= reading './lists/fromsource_sourcedistribution_main_coal_Packages'
-v0*=Getting packages...
-v1*=Shutting down aptmethods...
-v0*=Installing (and possibly deleting) packages...
EOF
rm -r conf dists pool db fakes addchecksums.rules old lists
......
......@@ -170,10 +170,6 @@ stdout
-v0*=Calculating packages to get...
-v3*= processing updates for 't|c|source'
-v5*= reading './lists/u_a_c_Sources'
stdout
-v0*=Getting packages...
-v1*=Shutting down aptmethods...
-v0*=Installing (and possibly deleting) packages...
EOF
rm -r conf db test lists pool dists
......
......@@ -934,10 +934,6 @@ stdout
-v3*= processing updates for 'test1|stupid|${FAKEARCHITECTURE}'
-v5*= reading './lists/_test1_stupid_${FAKEARCHITECTURE}_Test2toTest1_Test2toTest1_test2_stupid_${FAKEARCHITECTURE}_Packages'
-v5*= reading './lists/_test1_stupid_${FAKEARCHITECTURE}_Test2toTest1_Test2toTest1_test2_stupid_coal_Packages'
-v0*=Getting packages...
-v1=Freeing some memory...
-v1*=Shutting down aptmethods...
-v0*=Installing (and possibly deleting) packages...
EOF
checklog log1 < /dev/null
checknolog log2
......
......@@ -1757,6 +1757,23 @@ static retvalue updates_enqueue(struct downloadcache *cache,struct database *dat
* Step 11: install the missing packages *
* (missing files should have been downloaded first) *
****************************************************************************/
static bool isbigdelete(struct update_distribution *d) {
struct update_target *u, *v;
for( u = d->targets ; u != NULL ; u=u->next ) {
if( u->nothingnew || u->ignoredelete )
continue;
if( upgradelist_isbigdelete(u->upgradelist) ) {
d->distribution->omitted = true;
for( v = d->targets ; v != NULL ; v = v->next ) {
upgradelist_free(v->upgradelist);
v->upgradelist = NULL;
}
return true;
}
}
return false;
}
static void updates_from_callback(void *privdata, const char **rule_p, const char **from_p) {
struct update_index_connector *uindex = privdata;
......@@ -1968,6 +1985,7 @@ retvalue updates_update(struct database *database, struct update_distribution *d
struct update_distribution *d;
struct downloadcache *cache;
struct aptmethodrun *run IFSTUPIDCC(=NULL);
bool todo;
causingfile = NULL;
......@@ -1985,12 +2003,19 @@ retvalue updates_update(struct database *database, struct update_distribution *d
return result;
}
todo = false;
for( d=distributions ; d != NULL ; d=d->next) {
r = updates_readindices(stdout, database, d);
RET_UPDATE(result,r);
if( RET_WAS_ERROR(r) )
break;
if( global.onlysmalldeletes ) {
if( isbigdelete(d) )
continue;
}
r = updates_enqueue(cache, database, d);
if( RET_IS_OK(r) )
todo = true;
RET_UPDATE(result,r);
if( RET_WAS_ERROR(r) )
break;
......@@ -2000,9 +2025,15 @@ retvalue updates_update(struct database *database, struct update_distribution *d
RET_ENDUPDATE(result,r);
}
if( RET_WAS_ERROR(result) ) {
if( RET_WAS_ERROR(result) || !todo ) {
for( d=distributions ; d != NULL ; d=d->next) {
struct update_target *u;
if( d->distribution->omitted ) {
fprintf(stderr,
"Not processing updates for '%s' because of --onlysmalldeletes!\n",
d->distribution->codename);
} else if( RET_IS_OK(result) )
markdone(d);
for( u=d->targets ; u != NULL ; u=u->next ) {
upgradelist_free(u->upgradelist);
u->upgradelist = NULL;
......@@ -2038,6 +2069,8 @@ retvalue updates_update(struct database *database, struct update_distribution *d
printf("Installing (and possibly deleting) packages...\n");
for( d=distributions ; d != NULL ; d=d->next) {
if( d->distribution->omitted )
continue;
r = updates_install(database, d);
RET_UPDATE(result,r);
if( RET_WAS_ERROR(r) )
......@@ -2045,6 +2078,10 @@ retvalue updates_update(struct database *database, struct update_distribution *d
}
for( d=distributions ; d != NULL ; d=d->next) {
if( d->distribution->omitted ) {
fprintf(stderr, "Not processing updates for '%s' because of --onlysmalldeletes!\n",
d->distribution->codename);
} else
markdone(d);
}
logger_wait();
......
......@@ -593,6 +593,22 @@ retvalue upgradelist_predelete(struct upgradelist *upgrade, struct logger *logge
return result;
}
bool upgradelist_isbigdelete(const struct upgradelist *upgrade) {
struct package_data *pkg;
long long deleted = 0, all = 0;
if( upgrade->list == NULL )
return false;
for( pkg = upgrade->list ; pkg != NULL ; pkg = pkg->next ) {
if( pkg->version_in_use == NULL )
continue;
all++;
if( pkg->deleted )
deleted++;
}
return deleted >= 10 && all/deleted < 5;
}
retvalue upgradelist_install(struct upgradelist *upgrade, struct logger *logger, struct database *database, bool ignoredelete, void (*callback)(void *, const char **, const char **)){
struct package_data *pkg;
retvalue result,r;
......
......@@ -36,6 +36,8 @@ typedef retvalue enqueueaction(void *, struct database *, const struct checksums
/* request all wanted files refering the methods given before */
retvalue upgradelist_enqueue(struct upgradelist *, enqueueaction *, void *, struct database *);
bool upgradelist_isbigdelete(const struct upgradelist *);
retvalue upgradelist_install(struct upgradelist *upgrade, /*@null@*/struct logger *, struct database *, bool ignoredelete, void (*callback)(void *, const char **, const char **));
/* remove all packages that would either be removed or upgraded by an upgrade */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment