files.c 19.9 KB
Newer Older
Bernhard Link's avatar
Bernhard Link committed
1
/*  This file is part of "reprepro"
2
 *  Copyright (C) 2003,2004,2005,2006,2007 Bernhard R. Link
3
 *  This program is free software; you can redistribute it and/or modify
4
 *  it under the terms of the GNU General Public License version 2 as
Bernhard Link's avatar
Bernhard Link committed
5
 *  published by the Free Software Foundation.
6 7 8 9 10 11 12 13
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
Bernhard Link's avatar
Bernhard Link committed
14
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1301  USA
15
 */
Bernhard Link's avatar
Bernhard Link committed
16 17
#include <config.h>

Bernhard Link's avatar
Bernhard Link committed
18
#include <errno.h>
19
#include <assert.h>
Bernhard Link's avatar
Bernhard Link committed
20 21
#include <sys/stat.h>
#include <sys/types.h>
Bernhard Link's avatar
Bernhard Link committed
22
#include <ctype.h>
Bernhard Link's avatar
Bernhard Link committed
23
#include <stdio.h>
Bernhard Link's avatar
Bernhard Link committed
24
#include <stdlib.h>
25
#include <unistd.h>
Bernhard Link's avatar
Bernhard Link committed
26 27
#include <malloc.h>
#include <string.h>
Bernhard Link's avatar
Bernhard Link committed
28
#include "error.h"
29
#include "strlist.h"
30
#include "names.h"
Bernhard Link's avatar
Bernhard Link committed
31
#include "md5sum.h"
32
#include "checksums.h"
Bernhard Link's avatar
Bernhard Link committed
33 34 35
#include "dirs.h"
#include "names.h"
#include "files.h"
36
#include "copyfile.h"
Bernhard Link's avatar
Bernhard Link committed
37
#include "ignore.h"
38 39
#include "filelist.h"
#include "debfile.h"
40
#include "database_p.h"
Bernhard Link's avatar
Bernhard Link committed
41 42 43 44

extern int verbose;

/* Add file's md5sum to database */
45
retvalue files_add(struct database *database,const char *filekey,const char *md5sum) {
46
	return table_adduniqrecord(database->files, filekey, md5sum);
Bernhard Link's avatar
Bernhard Link committed
47
}
48
static retvalue files_get(struct database *database,const char *filekey,/*@out@*/char **md5sum) {
49
	return table_getrecord(database->files, filekey, md5sum);
50 51
}

52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
static retvalue files_get_checksums(struct database *database, const char *filekey, /*@out@*/struct checksums **checksums_p) {
	char *md5sum;
	retvalue r;

	r = table_getrecord(database->files, filekey, &md5sum);
	if( !RET_IS_OK(r) )
		return r;
	return checksums_set(checksums_p, md5sum);
}

static retvalue files_add_checksums(struct database *database, const char *filekey, const struct checksums *checksums) {
	retvalue r;
	char *md5sum;

	r = checksums_get(checksums, cs_md5sum, &md5sum);
	assert( r != RET_NOTHING);
	if( !RET_IS_OK(r) )
		return r;
	r = files_add(database, filekey, md5sum);
	free(md5sum);
	return r;
}
74

Bernhard Link's avatar
Bernhard Link committed
75
/* remove file's md5sum from database */
76
retvalue files_remove(struct database *database, const char *filekey, bool ignoremissing) {
77 78 79 80
	retvalue r;

	if( database->contents != NULL ) {
		(void)table_deleterecord(database->contents, filekey, true);
81
	}
82 83 84 85 86
	r = table_deleterecord(database->files, filekey, true);
	if( r == RET_NOTHING && !ignoremissing ) {
		fprintf(stderr, "To be forgotten filekey '%s' was not known.\n",
				filekey);
		return RET_ERROR_MISSING;
Bernhard Link's avatar
Bernhard Link committed
87
	}
88
	return r;
Bernhard Link's avatar
Bernhard Link committed
89 90
}

91
/* delete the file and remove its md5sum from database */
92
retvalue files_deleteandremove(struct database *database, const char *filekey, bool rmdirs, bool ignoreifnot) {
93 94 95 96
	int err,en;
	char *filename;
	retvalue r;

Bernhard Link's avatar
Bernhard Link committed
97
	if( interrupted() )
98
		return RET_ERROR_INTERRUPTED;
99
	if( verbose >= 1 )
100
		printf("deleting and forgetting %s\n",filekey);
101
	filename = files_calcfullfilename(database, filekey);
Bernhard Link's avatar
Bernhard Link committed
102
	if( filename == NULL )
103 104 105 106 107 108
		return RET_ERROR_OOM;
	err = unlink(filename);
	if( err != 0 ) {
		en = errno;
		r = RET_ERRNO(en);
		if( errno == ENOENT ) {
109 110
			if( !ignoreifnot )
				fprintf(stderr,"%s not found, forgetting anyway\n",filename);
111
		} else {
112 113
			fprintf(stderr, "error while unlinking %s: %m(%d)\n",
					filename, en);
114
			free(filename);
115 116
			return r;
		}
117 118 119
	} else if(rmdirs) {
		/* try to delete parent directories, until one gives
		 * errors (hopefully because it still contains files) */
120
		size_t fixedpartlen = strlen(database->mirrordir);
121 122 123 124
		char *p;

		while( (p = strrchr(filename,'/')) != NULL ) {
			/* do not try to remove parts of the mirrordir */
125
			if( (size_t)(p-filename) <= fixedpartlen+1 )
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
				break;
			*p ='\0';
			/* try to rmdir the directory, this will
			 * fail if there are still other files or directories
			 * in it: */
			err = rmdir(filename);
			if( err == 0 ) {
				if( verbose >= 1 ) {
					printf("removed now empty directory %s\n",filename);
				}
			} else {
				en = errno;
				if( en != ENOTEMPTY ) {
					//TODO: check here if only some
					//other error was first and it
					//is not empty so we do not have
					//to remove it anyway...
					fprintf(stderr,"ignoring error trying to rmdir %s: %m(%d)\n",filename,en);
				}
145 146
				/* parent directories will contain this one
				 * thus not be empty, in other words:
147 148 149 150 151 152
				 * everything's done */
				break;
			}
		}

	}
153
	free(filename);
154
	return files_remove(database, filekey, ignoreifnot);
155 156
}

157 158
/* check if file is already there (RET_NOTHING) or could be added (RET_OK)
 * or RET_ERROR_WRONG_MD5SUM if filekey is already there with different md5sum */
159
retvalue files_ready(struct database *database,const char *filekey,const char *md5sum) {
160 161
	retvalue r;
	const char *md5fromdatabase;
162

163 164 165 166 167 168 169 170 171 172 173 174
	r = table_gettemprecord(database->files, filekey,
			&md5fromdatabase, NULL);
	if( r == RET_NOTHING )
		return RET_OK;
	if( RET_WAS_ERROR(r) )
		return r;
	if( strcmp(md5sum, md5fromdatabase) != 0 ) {
		fprintf(stderr,
"File \"%s\" is already registered with other md5sum!\n"
"(expect: '%s', database:'%s')!\n",
				filekey, md5sum, md5fromdatabase);
		return RET_ERROR_WRONG_MD5;
175
	}
176
	return RET_NOTHING;
177 178
}

179 180
/* hardlink file with known checksums and add it to database */
retvalue files_hardlinkandadd(struct database *database, const char *tempfile, const char *filekey, const struct checksums *checksums) {
181
	retvalue r;
182 183 184

	/* an additional check to make sure nothing tricks us into
	 * overwriting it by another file */
185
	r = files_canadd(database, filekey, checksums);
186 187
	if( !RET_IS_OK(r) )
		return r;
188
	r = copyfile_hardlink(database->mirrordir, filekey, tempfile, NULL);
189 190
	if( RET_WAS_ERROR(r) )
		return r;
191

192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
	return files_add_checksums(database, filekey, checksums);
}

/* check for file in the database and if not found there, if it can be detected */
retvalue files_canadd(struct database *database, const char *filekey, const struct checksums *checksums) {
	retvalue r;
	const char *md5fromdatabase;

	r = table_gettemprecord(database->files, filekey,
			&md5fromdatabase, NULL);
	if( r == RET_NOTHING )
		return RET_OK;
	if( RET_WAS_ERROR(r) )
		return r;
	if( !checksums_matches(checksums, cs_md5sum, md5fromdatabase) ) {
		char *md5sum;

		r = checksums_get(checksums, cs_md5sum, &md5sum);
		assert( r != RET_NOTHING );
		if( RET_WAS_ERROR(r) ) {
			fprintf(stderr,
"File \"%s\" is already registered with other md5sum!\n"
"(database:'%s')!\n",
					filekey, md5fromdatabase);
			return RET_ERROR_WRONG_MD5;
		}
		fprintf(stderr,
"File \"%s\" is already registered with other md5sum!\n"
"(expect: '%s', database:'%s')!\n",
				filekey, md5sum, md5fromdatabase);
		free(md5sum);
		return RET_ERROR_WRONG_MD5;

	}
	// TODO: check other checksums here...
	return RET_NOTHING;
228
}
229

Bernhard Link's avatar
Bernhard Link committed
230
/* check for file in the database and if not found there, if it can be detected */
231
retvalue files_expect(struct database *database,const char *filekey,const char *md5sum) {
232 233 234 235 236
	retvalue r;
	char *filename;

	r = files_ready(database, filekey, md5sum);
	if( r == RET_NOTHING )
237
		return RET_OK;
238 239 240
	if( RET_WAS_ERROR(r) )
		return r;
	/* ready to add means missing, so have to look for the file itself: */
241

242 243 244 245 246 247 248
	filename = files_calcfullfilename(database, filekey);
	if( filename == NULL )
		return RET_ERROR_OOM;
	r = md5sum_ensure(filename, md5sum, true);
	free(filename);
	if( !RET_IS_OK(r) )
		return r;
Bernhard Link's avatar
Bernhard Link committed
249

250
	/* add found file to database */
251
	return files_add(database, filekey, md5sum);
Bernhard Link's avatar
Bernhard Link committed
252 253
}

254
/* check for several files in the database and in the pool if missing */
255
retvalue files_expectfiles(struct database *database,const struct strlist *filekeys,const struct strlist *md5sums) {
256 257 258 259 260 261 262
	int i;
	retvalue r;

	for( i = 0 ; i < filekeys->count ; i++ ) {
		const char *filekey = filekeys->values[i];
		const char *md5sum = md5sums->values[i];

263
		r = files_expect(database, filekey, md5sum);
264 265 266 267 268 269
		if( RET_WAS_ERROR(r) ) {
			return r;
		}
		if( r == RET_NOTHING ) {
			/* File missing */
			fprintf(stderr,"Missing file %s\n",filekey);
270
			return RET_ERROR_MISSING;
271 272 273 274 275 276
		}
	}
	return RET_OK;
}

/* print missing files */
277
retvalue files_printmissing(struct database *database,const struct strlist *filekeys,const struct strlist *md5sums,const struct strlist *origfiles) {
278 279 280 281 282 283 284 285 286
	int i;
	retvalue ret,r;

	ret = RET_NOTHING;
	for( i = 0 ; i < filekeys->count ; i++ ) {
		const char *filekey = filekeys->values[i];
		const char *md5sum = md5sums->values[i];
		const char *origfile = origfiles->values[i];

287
		r = files_expect(database, filekey, md5sum);
288 289 290 291 292
		if( RET_WAS_ERROR(r) ) {
			return r;
		}
		if( r == RET_NOTHING ) {
			/* File missing */
293 294 295
			fputs(origfile, stdout); putchar(' ');
			fputs(database->mirrordir, stdout); putchar('/');
			fputs(filekey, stdout); putchar('\n');
296 297 298 299 300 301 302
			RET_UPDATE(ret,RET_OK);
		} else
			RET_UPDATE(ret,r);
	}
	return ret;
}

Bernhard Link's avatar
Bernhard Link committed
303
/* dump out all information */
304
retvalue files_printmd5sums(struct database *database) {
305 306 307 308
	retvalue result, r;
	struct cursor *cursor;
	const char *filekey, *checksum;

309
	r = table_newglobalcursor(database->files, &cursor);
310 311
	if( !RET_IS_OK(r) )
		return r;
Bernhard Link's avatar
Bernhard Link committed
312
	result = RET_NOTHING;
313 314
	while( cursor_nexttemp(database->files, cursor,
				&filekey, &checksum) ) {
Bernhard Link's avatar
Bernhard Link committed
315
		result = RET_OK;
316 317
		fputs(filekey, stdout);putchar(' ');
		fputs(checksum, stdout);putchar('\n');
Bernhard Link's avatar
Bernhard Link committed
318
	}
319 320
	r = cursor_close(database->files, cursor);
	RET_ENDUPDATE(result, r);
Bernhard Link's avatar
Bernhard Link committed
321
	return result;
Bernhard Link's avatar
Bernhard Link committed
322
}
323 324

/* callback for each registered file */
325
retvalue files_foreach(struct database *database,per_file_action action,void *privdata) {
326 327 328
	retvalue result, r;
	struct cursor *cursor;
	const char *filekey, *checksum;
329

330
	r = table_newglobalcursor(database->files, &cursor);
331 332
	if( !RET_IS_OK(r) )
		return r;
333
	result = RET_NOTHING;
334 335 336
	while( cursor_nexttemp(database->files, cursor,
				&filekey, &checksum) ) {
		r = action(privdata, filekey, checksum);
337 338
		RET_UPDATE(result,r);
	}
339 340
	r = cursor_close(database->files, cursor);
	RET_ENDUPDATE(result, r);
341 342
	return result;
}
343

Bernhard Link's avatar
Bernhard Link committed
344
static retvalue getfilesize(/*@out@*/off_t *s,const char *md5sum) {
Bernhard Link's avatar
Bernhard Link committed
345 346 347
	const char *p;

	p = md5sum;
Bernhard Link's avatar
Bernhard Link committed
348
	while( *p != '\0' && !xisspace(*p) ) {
Bernhard Link's avatar
Bernhard Link committed
349 350
		p++;
	}
Bernhard Link's avatar
Bernhard Link committed
351 352
	if( *p != '\0' ) {
		while( *p != '\0' && xisspace(*p) )
Bernhard Link's avatar
Bernhard Link committed
353
			p++;
Bernhard Link's avatar
Bernhard Link committed
354
		if( *p != '\0' ) {
Bernhard Link's avatar
Bernhard Link committed
355 356 357
			*s = (off_t)atoll(p);
			return RET_OK;
		}
358
	}
Bernhard Link's avatar
Bernhard Link committed
359 360 361 362
	fprintf(stderr,"Strange md5sum as missing space: '%s'\n",md5sum);
	return RET_ERROR;
}

363 364 365 366
retvalue files_checkpool(struct database *database, bool fast) {
	retvalue result, r;
	struct cursor *cursor;
	const char *filekey, *md5sumexpected;
Bernhard Link's avatar
Bernhard Link committed
367 368
	char *fullfilename;

369
	r = table_newglobalcursor(database->files, &cursor);
370 371 372 373 374 375 376 377 378 379 380 381 382 383
	if( !RET_IS_OK(r) )
		return r;
	result = RET_NOTHING;
	while( cursor_nexttemp(database->files, cursor,
				&filekey, &md5sumexpected) ) {
		fullfilename = files_calcfullfilename(database, filekey);
		if( fullfilename == NULL ) {
			result = RET_ERROR_OOM;
			break;
		}
		if( fast ) {
			struct stat s;
			int i;
			off_t expectedsize;
Bernhard Link's avatar
Bernhard Link committed
384

385 386 387 388 389 390 391
			r = getfilesize(&expectedsize, md5sumexpected);
			assert( r != RET_NOTHING );
			if( RET_WAS_ERROR(r) ) {
				free(fullfilename);
				RET_UPDATE(result, r);
				continue;
			}
Bernhard Link's avatar
Bernhard Link committed
392 393 394
			i = stat(fullfilename,&s);
			if( i < 0 ) {
				fprintf(stderr,
395
"Error checking status of '%s': %m\n", fullfilename);
Bernhard Link's avatar
Bernhard Link committed
396 397 398 399 400 401 402 403 404 405 406
				r = RET_ERROR_MISSING;
			} else {
				if( !S_ISREG(s.st_mode)) {
					fprintf(stderr,
"Not a regular file: '%s'\n",fullfilename);
					r = RET_ERROR;
				} else if( s.st_size != expectedsize ) {
					fprintf(stderr,
"WRONG SIZE of '%s': expected %lld(from '%s') found %lld\n",
						fullfilename,
						(long long)expectedsize,
407 408
						md5sumexpected,
						(long long)s.st_size);
Bernhard Link's avatar
Bernhard Link committed
409 410 411 412 413
					r = RET_ERROR;
				} else
					r = RET_OK;
			}
		} else {
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
			char *realmd5sum;

			r = md5sum_read(fullfilename, &realmd5sum);
			if( r == RET_NOTHING ) {
				fprintf(stderr,"Missing file '%s'!\n",
						fullfilename);
				r = RET_ERROR_MISSING;
			} else if( RET_IS_OK(r) ) {
				if( strcmp(realmd5sum,md5sumexpected) != 0 ) {
					fprintf(stderr,
"WRONG MD5SUM of '%s': found '%s' expected '%s'\n",
						fullfilename, realmd5sum,
						md5sumexpected);
					r = RET_ERROR_WRONG_MD5;
				}
				free(realmd5sum);
Bernhard Link's avatar
Bernhard Link committed
430 431
			}
		}
432 433
		free(fullfilename);
		RET_UPDATE(result,r);
Bernhard Link's avatar
Bernhard Link committed
434
	}
435 436 437
	r = cursor_close(database->files, cursor);
	RET_ENDUPDATE(result, r);
	return result;
Bernhard Link's avatar
Bernhard Link committed
438 439
}

440
retvalue files_detect(struct database *database,const char *filekey) {
Bernhard Link's avatar
Bernhard Link committed
441 442 443
	char *md5sum;
	char *fullfilename;
	retvalue r;
444

445
	fullfilename = files_calcfullfilename(database, filekey);
Bernhard Link's avatar
Bernhard Link committed
446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
	if( fullfilename == NULL )
		return RET_ERROR_OOM;
	r = md5sum_read(fullfilename,&md5sum);
	if( r == RET_NOTHING ) {
		fprintf(stderr,"Error opening '%s'!\n",fullfilename);
		r = RET_ERROR_MISSING;
	}
	if( RET_WAS_ERROR(r) ) {
		free(fullfilename);
		return r;
	}
	if( verbose > 20 ) {
		fprintf(stderr,"Md5sum of '%s' is '%s'.\n",fullfilename,md5sum);
	}
	free(fullfilename);
461
	r = files_add(database, filekey, md5sum);
Bernhard Link's avatar
Bernhard Link committed
462 463 464 465 466 467
	free(md5sum);
	return r;


}

468
/* Include a given file into the pool. */
469
retvalue files_include(struct database *database,const char *sourcefilename,const char *filekey, const char *md5sum, char **calculatedmd5sum, int delete) {
470 471 472 473
	retvalue r;
	char *md5sumfound;

	if( md5sum != NULL ) {
474
		r = files_expect(database, filekey, md5sum);
475 476 477 478
		if( RET_WAS_ERROR(r) )
			return r;
		if( RET_IS_OK(r) ) {
			if( delete >= D_MOVE ) {
479
				deletefile(sourcefilename);
480
			}
Bernhard Link's avatar
Bernhard Link committed
481 482 483 484 485 486 487
			if( calculatedmd5sum != NULL ) {
				char *n = strdup(md5sum);
				if( n == NULL )
					return RET_ERROR_OOM;
				*calculatedmd5sum = n;
			}
			return RET_OK;
488 489 490 491
		}
	} else {
		char *md5indatabase,*md5offile;

492
		r = files_get(database, filekey, &md5indatabase);
493
		if( RET_WAS_ERROR(r) )
494 495 496 497 498 499 500
			return r;
		if( RET_IS_OK(r) ) {
			if( delete == D_INPLACE ) {
				if( calculatedmd5sum != NULL )
					*calculatedmd5sum = md5indatabase;
				else
					free(md5indatabase);
Bernhard Link's avatar
Bernhard Link committed
501
				return RET_OK;
502
			}
503 504 505 506

			r = md5sum_read(sourcefilename, &md5offile);
			if( r == RET_NOTHING )
				r = RET_ERROR_MISSING;
507 508 509 510 511 512 513 514 515 516 517 518
			if( RET_WAS_ERROR(r) ) {
				free(md5indatabase);
				return r;
			}
			if( strcmp(md5indatabase,md5offile) != 0 ) {
				fprintf(stderr,"File \"%s\" is already registered with other md5sum!\n(file: '%s', database:'%s')!\n",filekey,md5offile,md5indatabase);
				free(md5offile);
				free(md5indatabase);
				return RET_ERROR_WRONG_MD5;
			} else {
				// The file has the md5sum we know already.
				if( delete >= D_MOVE ) {
519
					deletefile(sourcefilename);
520 521 522 523 524 525 526 527 528 529
				}
				if( calculatedmd5sum != NULL )
					*calculatedmd5sum = md5indatabase;
				else
					free(md5indatabase);
				free(md5offile);
				return RET_NOTHING;
			}
		}
	}
Bernhard Link's avatar
Bernhard Link committed
530
	if( sourcefilename == NULL ) {
531 532
		fprintf(stderr,"Unable to find %s/%s!\n",
				database->mirrordir, filekey);
533 534 535
		if( delete == D_INPLACE ) {
			fprintf(stderr,"Perhaps you forgot to give dpkg-buildpackage the -sa option,\n or you cound try --ignore=missingfile\n");
		}
536
		return RET_ERROR_MISSING;
Bernhard Link's avatar
Bernhard Link committed
537
	} if( delete == D_INPLACE ) {
538 539 540
		if( IGNORING("Looking around if it is elsewhere", "To look around harder, ",missingfile,"Unable to find %s/%s!\n", database->mirrordir, filekey))
			r = copyfile_copy(database->mirrordir, filekey,
					sourcefilename, md5sum, &md5sumfound);
Bernhard Link's avatar
Bernhard Link committed
541 542 543 544
		else
			r = RET_ERROR_MISSING;
		if( RET_WAS_ERROR(r) )
			return r;
545
	} else if( delete == D_COPY ) {
546 547
		r = copyfile_copy(database->mirrordir, filekey, sourcefilename,
				md5sum, &md5sumfound);
548 549 550 551
		if( RET_WAS_ERROR(r) )
			return r;
	} else {
		assert( delete >= D_MOVE );
552 553
		r = copyfile_move(database->mirrordir, filekey, sourcefilename,
				md5sum, &md5sumfound);
554 555 556 557
		if( RET_WAS_ERROR(r) )
			return r;
	}

558
	r = files_add(database, filekey, md5sumfound);
559 560 561 562 563 564
	if( RET_WAS_ERROR(r) ) {
		free(md5sumfound);
		return r;
	}
	if( calculatedmd5sum != NULL )
		*calculatedmd5sum = md5sumfound;
565
	else
566 567 568 569 570
		free(md5sumfound);
	return RET_OK;
}

/* same as above, but use sourcedir/basename instead of sourcefilename */
571
retvalue files_includefile(struct database *database,const char *sourcedir,const char *basename, const char *filekey, const char *md5sum, char **calculatedmd5sum, int delete) {
572 573 574
	char *sourcefilename;
	retvalue r;

Bernhard Link's avatar
Bernhard Link committed
575 576 577 578 579 580 581 582
	if( sourcedir == NULL ) {
		assert(delete == D_INPLACE);
		sourcefilename = NULL;
	} else {
		sourcefilename = calc_dirconcat(sourcedir,basename);
		if( sourcefilename == NULL )
			return RET_ERROR_OOM;
	}
583 584
	r = files_include(database, sourcefilename, filekey, md5sum,
			calculatedmd5sum, delete);
585 586
	free(sourcefilename);
	return r;
587

588 589
}

590
retvalue files_regenerate_filelist(struct database *database, bool reread) {
591
	struct cursor *cursor;
592
	retvalue result,r;
593
	const char *filekey, *checksum;
594

595
	assert( database->contents != NULL );
596

597
	r = table_newglobalcursor(database->files, &cursor);
598 599
	if( !RET_IS_OK(r) )
		return r;
600
	result = RET_NOTHING;
601 602 603
	while( cursor_nexttemp(database->files, cursor,
				&filekey, &checksum) ) {
		size_t l = strlen(filekey);
604 605
		if( l > 4 && strcmp(filekey+l-4,".deb") == 0 ) {

606 607
			if( reread || !table_recordexists(
					database->contents, filekey) ) {
608 609
				char *debfilename;
				char *filelist;
610
				size_t fls;
611

612 613
				debfilename = files_calcfullfilename(database,
						filekey);
614
				if( debfilename == NULL ) {
615 616
					result = r;
					break;
617 618
				}

619
				r = getfilelist(&filelist, &fls, debfilename);
620 621 622 623 624 625 626 627 628 629 630 631
				free(debfilename);
				if( RET_IS_OK(r) ) {
					if( verbose > 0 )
						puts(filekey);
					if( verbose > 6 ) {
						const char *p = filelist;
						while( *p != '\0' ) {
							putchar(' ');
							puts(p);
							p += strlen(p)+1;
						}
					}
632
					r = table_adduniqsizedrecord(
633 634
							database->contents,
							filekey, filelist, fls,
635
							true, true);
636 637
					free(filelist);
				}
638
				RET_UPDATE(result,r);
639
				if( RET_WAS_ERROR(r) ) {
640
					break;
641 642 643 644
				}
			}
		}
	}
645 646
	r = cursor_close(database->files, cursor);
	RET_ENDUPDATE(result, r);
647 648
	return result;
}
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737

/* Include a yet unknown file into the pool */
retvalue files_preinclude(struct database *database, const char *sourcefilename, const char *filekey, struct checksums **checksums_p) {
	retvalue r;
	struct checksums *checksums, *realchecksums;
	bool improves;
	char *fullfilename;

	r = files_get_checksums(database, filekey, &checksums);
	if( RET_WAS_ERROR(r) )
		return r;
	if( RET_IS_OK(r) ) {
		r = checksums_read(sourcefilename, &realchecksums);
		if( r == RET_NOTHING )
			r = RET_ERROR_MISSING;
		if( RET_WAS_ERROR(r) ) {
			checksums_free(checksums);
			return r;
		}
		if( !checksums_check(checksums, realchecksums, &improves) ) {
			fprintf(stderr,
"ERROR: '%s' cannot be included as '%s'.\n"
"Already existing files can only be included again, if they are the same, but:\n",
				sourcefilename, filekey);
			checksums_printdifferences(stderr, checksums, realchecksums);
			checksums_free(checksums);
			checksums_free(realchecksums);
			return RET_ERROR_WRONG_MD5;
		}
		if( improves ) {
			r = checksums_combine(&checksums, realchecksums);
			if( RET_WAS_ERROR(r) ) {
				checksums_free(checksums);
				return r;
			}
			/* TODO:
			r = files_add_improved(database, filekey, checksums);
			if( RET_WAS_ERROR(r) ) {
				checksums_free(checksums);
				return r;
			}*/
		} else
			checksums_free(realchecksums);
		// args, this breaks retvalue semantics!
		*checksums_p = checksums;
		return RET_NOTHING;
	}
	assert( sourcefilename != NULL );
	fullfilename = files_calcfullfilename(database, filekey);
	if( fullfilename == NULL )
		return RET_ERROR_OOM;
	r = checksums_copyfile(fullfilename, sourcefilename, &checksums);
	if( r == RET_ERROR_EXIST ) {
		// TODO: deal with already existing files!
		fprintf(stderr, "File '%s' does already exist!\n", fullfilename);
	}
	if( r == RET_NOTHING ) {
		fprintf(stderr, "Could not open '%s'!\n", sourcefilename);
		r = RET_ERROR_MISSING;
	}
	if( RET_WAS_ERROR(r) ) {
		free(fullfilename);
		return r;
	}
	free(fullfilename);

	r = files_add_checksums(database, filekey, checksums);
	if( RET_WAS_ERROR(r) ) {
		checksums_free(checksums);
		return r;
	}
	*checksums_p = checksums;
	return RET_OK;
}

retvalue files_preincludefile(struct database *database, const char *sourcedir,const char *basename, const char *filekey, struct checksums **checksums_p) {
	char *sourcefilename;
	retvalue r;

	assert( sourcedir != NULL );
	sourcefilename = calc_dirconcat(sourcedir, basename);
	if( sourcefilename == NULL )
		return RET_ERROR_OOM;
	r = files_preinclude(database, sourcefilename, filekey, checksums_p);
	free(sourcefilename);
	return r;

}