incoming.c 71.5 KB
Newer Older
1
/*  This file is part of "reprepro"
2
 *  Copyright (C) 2006,2007,2008,2009,2010 Bernhard R. Link
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 2 as
 *  published by the Free Software Foundation.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1301  USA
 */
#include <config.h>

#include <errno.h>
#include <assert.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <getopt.h>
#include <string.h>
#include <strings.h>
#include <fcntl.h>
#include <signal.h>
#include <dirent.h>
29
#include <time.h>
30
#include <sys/stat.h>
31 32 33
#include "error.h"
#include "ignore.h"
#include "mprintf.h"
34
#include "filecntl.h"
35 36 37
#include "strlist.h"
#include "dirs.h"
#include "names.h"
38
#include "checksums.h"
39 40 41
#include "chunks.h"
#include "target.h"
#include "signature.h"
42
#include "binaries.h"
43
#include "sources.h"
44 45 46
#include "dpkgversions.h"
#include "uploaderslist.h"
#include "guesscomponent.h"
Bernhard Link's avatar
Bernhard Link committed
47
#include "log.h"
48
#include "override.h"
49
#include "tracking.h"
50
#include "incoming.h"
51
#include "files.h"
Bernhard Link's avatar
Bernhard Link committed
52
#include "configparser.h"
53
#include "byhandhook.h"
54 55
#include "changes.h"

Bernhard Link's avatar
Bernhard Link committed
56 57 58 59 60
enum permitflags {
	/* do not error out on unused files */
	pmf_unused_files = 0,
	/* do not error out if there already is a newer package */
	pmf_oldpackagenewer,
61 62
	/* do not error out if there there are unadvertised binary files */
	pmf_unlistedbinaries,
Bernhard Link's avatar
Bernhard Link committed
63 64 65 66 67 68 69 70 71 72 73
	pmf_COUNT /* must be last */
};
enum cleanupflags {
	/* delete everything referenced by a .changes file
	 * when it is not accepted */
	cuf_on_deny = 0,
	/* check owner when deleting on_deny */
	cuf_on_deny_check_owner,
	/* delete everything referenced by a .changes on errors
	 * after accepting that .changes file*/
	cuf_on_error,
74
	/* delete unused files after successfully
Bernhard Link's avatar
Bernhard Link committed
75 76
	 * processing the used ones */
	cuf_unused_files,
77 78
	/* same but restricted to .buildinfo files */
	cuf_unused_buildinfo_files,
Bernhard Link's avatar
Bernhard Link committed
79 80
	cuf_COUNT /* must be last */
};
81 82 83 84
enum optionsflags {
	/* only put _all.deb comes with those of some architecture,
	 * only put in those architectures */
	iof_limit_arch_all = 0,
85
	/* allow .changes file to specify multiple distributions */
86 87 88
	iof_multiple_distributions,
	iof_COUNT /* must be last */
};
Bernhard Link's avatar
Bernhard Link committed
89

90 91
struct incoming {
	/* by incoming_parse: */
92
	char *name;
93
	char *directory;
94
	char *morguedir;
95
	char *tempdir;
96
	char *logdir;
97 98 99 100 101
	struct strlist allow;
	struct distribution **allow_into;
	struct distribution *default_into;
	/* by incoming_prepare: */
	struct strlist files;
102 103 104 105
	bool *processed;
	bool *delete;
	bool permit[pmf_COUNT];
	bool cleanup[cuf_COUNT];
106
	bool options[iof_COUNT];
Bernhard Link's avatar
Bernhard Link committed
107
	/* only to ease parsing: */
108
	const char *filename; /* only valid while parsing! */
109
	size_t lineno;
110
};
111
#define BASENAME(i, ofs) (i)->files.values[ofs]
112 113
/* the changes file is always the first one listed */
#define changesfile(c) (c->files)
114 115

static void incoming_free(/*@only@*/ struct incoming *i) {
116
	if (i == NULL)
117
		return;
Bernhard Link's avatar
Bernhard Link committed
118
	free(i->name);
119
	free(i->morguedir);
120
	free(i->tempdir);
121
	free(i->logdir);
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
	free(i->directory);
	strlist_done(&i->allow);
	free(i->allow_into);
	strlist_done(&i->files);
	free(i->processed);
	free(i->delete);
	free(i);
}

static retvalue incoming_prepare(struct incoming *i) {
	DIR *dir;
	struct dirent *ent;
	retvalue r;
	int ret;

	/* TODO: decide whether to clean this directory first ... */
	r = dirs_make_recursive(i->tempdir);
139
	if (RET_WAS_ERROR(r))
140 141
		return r;
	dir = opendir(i->directory);
142
	if (dir == NULL) {
143
		int e = errno;
144 145
		fprintf(stderr, "Cannot scan '%s': %s\n",
				i->directory, strerror(e));
146 147
		return RET_ERRNO(e);
	}
148 149
	while ((ent = readdir(dir)) != NULL) {
		if (ent->d_name[0] == '.')
Bernhard Link's avatar
Bernhard Link committed
150 151 152 153
			continue;
		/* this should be impossible to hit.
		 * but given utf-8 encoding filesystems and
		 * overlong slashes, better check than be sorry */
154
		if (strchr(ent->d_name, '/') != NULL)
Bernhard Link's avatar
Bernhard Link committed
155
			continue;
156
		r = strlist_add_dup(&i->files, ent->d_name) ;
157
		if (RET_WAS_ERROR(r)) {
158
			(void)closedir(dir);
159 160 161 162
			return r;
		}
	}
	ret = closedir(dir);
163
	if (ret != 0) {
164
		int e = errno;
165 166
		fprintf(stderr, "Error scanning '%s': %s\n",
				i->directory, strerror(e));
167 168
		return RET_ERRNO(e);
	}
169 170
	i->processed = nzNEW(i->files.count, bool);
	if (FAILEDTOALLOC(i->processed))
171
		return RET_ERROR_OOM;
172 173
	i->delete = nzNEW(i->files.count, bool);
	if (FAILEDTOALLOC(i->delete))
174 175 176 177
		return RET_ERROR_OOM;
	return RET_OK;
}

Bernhard Link's avatar
Bernhard Link committed
178
struct read_incoming_data {
179 180
	/*@temp@*/const char *name;
	/*@temp@*/struct distribution *distributions;
181 182 183 184 185 186 187
	struct incoming *i;
};

static retvalue translate(struct distribution *distributions, struct strlist *names, struct distribution ***r) {
	struct distribution **d;
	int j;

188 189
	d = nzNEW(names->count, struct distribution *);
	if (FAILEDTOALLOC(d))
190
		return RET_ERROR_OOM;
191
	for (j = 0 ; j < names->count ; j++) {
192
		d[j] = distribution_find(distributions, names->values[j]);
193
		if (d[j] == NULL) {
194 195 196 197 198 199 200 201
			free(d);
			return RET_ERROR;
		}
	}
	*r = d;
	return RET_OK;
}

Bernhard Link's avatar
Bernhard Link committed
202
CFstartparse(incoming) {
203
	CFstartparseVAR(incoming, result_p);
204 205
	struct incoming *i;

206 207
	i = zNEW(struct incoming);
	if (FAILEDTOALLOC(i))
208
		return RET_ERROR_OOM;
Bernhard Link's avatar
Bernhard Link committed
209 210 211 212 213
	*result_p = i;
	return RET_OK;
}

CFfinishparse(incoming) {
214
	CFfinishparseVARS(incoming, i, last, d);
215

216
	if (!complete || strcmp(i->name, d->name) != 0) {
Bernhard Link's avatar
Bernhard Link committed
217 218
		incoming_free(i);
		return RET_NOTHING;
219
	}
220 221
	if (d->i != NULL) {
		fprintf(stderr,
222
"Multiple definitions of '%s': first started at line %u of %s, second at line %u of %s!\n",
Bernhard Link's avatar
Bernhard Link committed
223
				d->name,
224 225
				(unsigned int)d->i->lineno, d->i->filename,
				config_firstline(iter), config_filename(iter));
226
		incoming_free(i);
Bernhard Link's avatar
Bernhard Link committed
227 228 229
		incoming_free(d->i);
		d->i = NULL;
		return RET_ERROR;
230
	}
231
	if (i->logdir != NULL && i->logdir[0] != '/') {
232
		char *n = calc_dirconcat(global.basedir, i->logdir);
233
		if (FAILEDTOALLOC(n)) {
234 235 236 237 238 239
			incoming_free(i);
			return RET_ERROR_OOM;
		}
		free(i->logdir);
		i->logdir = n;
	}
240
	if (i->morguedir != NULL && i->morguedir[0] != '/') {
241
		char *n = calc_dirconcat(global.basedir, i->morguedir);
242
		if (FAILEDTOALLOC(n)) {
243 244 245 246 247 248
			incoming_free(i);
			return RET_ERROR_OOM;
		}
		free(i->morguedir);
		i->morguedir = n;
	}
249
	if (i->tempdir[0] != '/') {
250
		char *n = calc_dirconcat(global.basedir, i->tempdir);
251
		if (FAILEDTOALLOC(n)) {
252 253 254 255 256 257
			incoming_free(i);
			return RET_ERROR_OOM;
		}
		free(i->tempdir);
		i->tempdir = n;
	}
258
	if (i->directory[0] != '/') {
259
		char *n = calc_dirconcat(global.basedir, i->directory);
260
		if (FAILEDTOALLOC(n)) {
261 262 263 264 265 266
			incoming_free(i);
			return RET_ERROR_OOM;
		}
		free(i->directory);
		i->directory = n;
	}
267
	if (i->default_into == NULL && i->allow.count == 0) {
Bernhard Link's avatar
Bernhard Link committed
268
		fprintf(stderr,
Bernhard Link's avatar
Bernhard Link committed
269
"There is neither an 'Allow' nor a 'Default' definition in rule '%s'\n"
Bernhard Link's avatar
Bernhard Link committed
270 271 272 273 274
"(starting at line %u, ending at line %u of %s)!\n"
"Aborting as nothing would be let in.\n",
				d->name,
				config_firstline(iter), config_line(iter),
				config_filename(iter));
275
			incoming_free(i);
Bernhard Link's avatar
Bernhard Link committed
276
			return RET_ERROR;
277
	}
278
	if (i->morguedir != NULL && !i->cleanup[cuf_on_deny]
279
			&& !i->cleanup[cuf_on_error]
280
			&& !i->cleanup[cuf_unused_buildinfo_files]
281
			&& !i->cleanup[cuf_unused_files]) {
282 283 284 285 286 287 288
		fprintf(stderr,
"Warning: There is a 'MorgueDir' but no 'Cleanup' to act on in rule '%s'\n"
"(starting at line %u, ending at line %u of %s)!\n",
				d->name,
				config_firstline(iter), config_line(iter),
				config_filename(iter));
	}
Bernhard Link's avatar
Bernhard Link committed
289 290

	d->i = i;
291
	i->filename = config_filename(iter);
Bernhard Link's avatar
Bernhard Link committed
292 293 294 295 296 297
	i->lineno = config_firstline(iter);
	/* only suppreses the last unused warning: */
	*last = i;
	return RET_OK;
}

298 299
CFSETPROC(incoming, default) {
	CFSETPROCVARS(incoming, i, d);
Bernhard Link's avatar
Bernhard Link committed
300 301 302
	char *default_into;
	retvalue r;

303
	r = config_getonlyword(iter, headername, NULL, &default_into);
304 305
	assert (r != RET_NOTHING);
	if (RET_WAS_ERROR(r))
306
		return r;
Bernhard Link's avatar
Bernhard Link committed
307 308
	i->default_into = distribution_find(d->distributions, default_into);
	free(default_into);
309
	return (i->default_into == NULL)?RET_ERROR:RET_OK;
Bernhard Link's avatar
Bernhard Link committed
310 311
}

312 313
CFSETPROC(incoming, allow) {
	CFSETPROCVARS(incoming, i, d);
Bernhard Link's avatar
Bernhard Link committed
314 315 316 317
	struct strlist allow_into;
	retvalue r;

	r = config_getsplitwords(iter, headername, &i->allow, &allow_into);
318 319
	assert (r != RET_NOTHING);
	if (RET_WAS_ERROR(r))
320
		return r;
321
	assert (i->allow.count == allow_into.count);
Bernhard Link's avatar
Bernhard Link committed
322 323
	r = translate(d->distributions, &allow_into, &i->allow_into);
	strlist_done(&allow_into);
324
	if (RET_WAS_ERROR(r))
325
		return r;
326 327 328
	return RET_OK;
}

329 330
CFSETPROC(incoming, permit) {
	CFSETPROCVARS(incoming, i, d);
331
	static const struct constant permitconstants[] = {
Bernhard Link's avatar
Bernhard Link committed
332 333
		{ "unused_files",	pmf_unused_files},
		{ "older_version",	pmf_oldpackagenewer},
334
		{ "unlisted_binaries",	pmf_unlistedbinaries},
Bernhard Link's avatar
Bernhard Link committed
335 336 337 338 339 340
		/* not yet implemented:
		   { "downgrade",		pmf_downgrade},
		 */
		{ NULL, -1}
	};

341
	if (IGNORABLE(unknownfield))
Bernhard Link's avatar
Bernhard Link committed
342
		return config_getflags(iter, headername, permitconstants,
343
				i->permit, true, "");
344
	else if (i->name == NULL)
Bernhard Link's avatar
Bernhard Link committed
345
		return config_getflags(iter, headername, permitconstants,
346
				i->permit, false,
Bernhard Link's avatar
Bernhard Link committed
347
"\n(try put Name: before Permit: to ignore if it is from the wrong rule");
348
	else if (strcmp(i->name, d->name) != 0)
Bernhard Link's avatar
Bernhard Link committed
349
		return config_getflags(iter, headername, permitconstants,
350
				i->permit, true,
Bernhard Link's avatar
Bernhard Link committed
351 352 353
" (but not within the rule we are intrested in.)");
	else
		return config_getflags(iter, headername, permitconstants,
354
				i->permit, false,
Bernhard Link's avatar
Bernhard Link committed
355 356 357 358
" (use --ignore=unknownfield to ignore this)\n");

}

359 360
CFSETPROC(incoming, cleanup) {
	CFSETPROCVARS(incoming, i, d);
361
	static const struct constant cleanupconstants[] = {
Bernhard Link's avatar
Bernhard Link committed
362
		{ "unused_files", cuf_unused_files},
363
		{ "unused_buildinfo_files", cuf_unused_buildinfo_files},
Bernhard Link's avatar
Bernhard Link committed
364 365 366 367 368 369 370 371
		{ "on_deny", cuf_on_deny},
		/* not yet implemented
		{ "on_deny_check_owner", cuf_on_deny_check_owner},
		 */
		{ "on_error", cuf_on_error},
		{ NULL, -1}
	};

372
	if (IGNORABLE(unknownfield))
Bernhard Link's avatar
Bernhard Link committed
373
		return config_getflags(iter, headername, cleanupconstants,
374
				i->cleanup, true, "");
375
	else if (i->name == NULL)
Bernhard Link's avatar
Bernhard Link committed
376
		return config_getflags(iter, headername, cleanupconstants,
377
				i->cleanup, false,
Bernhard Link's avatar
Bernhard Link committed
378
"\n(try put Name: before Cleanup: to ignore if it is from the wrong rule");
379
	else if (strcmp(i->name, d->name) != 0)
Bernhard Link's avatar
Bernhard Link committed
380
		return config_getflags(iter, headername, cleanupconstants,
381
				i->cleanup, true,
Bernhard Link's avatar
Bernhard Link committed
382 383 384
" (but not within the rule we are intrested in.)");
	else
		return config_getflags(iter, headername, cleanupconstants,
385
				i->cleanup, false,
Bernhard Link's avatar
Bernhard Link committed
386 387 388
" (use --ignore=unknownfield to ignore this)\n");
}

389 390
CFSETPROC(incoming, options) {
	CFSETPROCVARS(incoming, i, d);
391
	static const struct constant optionsconstants[] = {
392 393 394 395 396
		{ "limit_arch_all", iof_limit_arch_all},
		{ "multiple_distributions", iof_multiple_distributions},
		{ NULL, -1}
	};

397
	if (IGNORABLE(unknownfield))
398 399
		return config_getflags(iter, headername, optionsconstants,
				i->options, true, "");
400
	else if (i->name == NULL)
401 402 403
		return config_getflags(iter, headername, optionsconstants,
				i->options, false,
"\n(try put Name: before Options: to ignore if it is from the wrong rule");
404
	else if (strcmp(i->name, d->name) != 0)
405 406 407 408 409 410 411 412 413
		return config_getflags(iter, headername, optionsconstants,
				i->options, true,
" (but not within the rule we are intrested in.)");
	else
		return config_getflags(iter, headername, optionsconstants,
				i->options, false,
" (use --ignore=unknownfield to ignore this)\n");
}

Bernhard Link's avatar
Bernhard Link committed
414
CFvalueSETPROC(incoming, name)
415
CFdirSETPROC(incoming, logdir)
Bernhard Link's avatar
Bernhard Link committed
416
CFdirSETPROC(incoming, tempdir)
417
CFdirSETPROC(incoming, morguedir)
Bernhard Link's avatar
Bernhard Link committed
418
CFdirSETPROC(incoming, directory)
419
CFtruthSETPROC2(incoming, multiple, options[iof_multiple_distributions])
Bernhard Link's avatar
Bernhard Link committed
420 421 422 423 424

static const struct configfield incomingconfigfields[] = {
	CFr("Name", incoming, name),
	CFr("TempDir", incoming, tempdir),
	CFr("IncomingDir", incoming, directory),
425
	CF("MorgueDir", incoming, morguedir),
Bernhard Link's avatar
Bernhard Link committed
426 427 428
	CF("Default", incoming, default),
	CF("Allow", incoming, allow),
	CF("Multiple", incoming, multiple),
429
	CF("Options", incoming, options),
Bernhard Link's avatar
Bernhard Link committed
430
	CF("Cleanup", incoming, cleanup),
431 432
	CF("Permit", incoming, permit),
	CF("Logdir", incoming, logdir)
Bernhard Link's avatar
Bernhard Link committed
433 434
};

435
static retvalue incoming_init(struct distribution *distributions, const char *name, /*@out@*/struct incoming **result) {
436
	retvalue r;
Bernhard Link's avatar
Bernhard Link committed
437
	struct read_incoming_data imports;
438 439 440 441 442

	imports.name = name;
	imports.distributions = distributions;
	imports.i = NULL;

443
	r = configfile_parse("incoming", IGNORABLE(unknownfield),
Bernhard Link's avatar
Bernhard Link committed
444
			startparseincoming, finishparseincoming,
445
			"incoming rule",
Bernhard Link's avatar
Bernhard Link committed
446 447
			incomingconfigfields, ARRAYCOUNT(incomingconfigfields),
			&imports);
448
	if (RET_WAS_ERROR(r))
449
		return r;
450 451 452
	if (imports.i == NULL) {
		fprintf(stderr,
"No definition for '%s' found in '%s/incoming'!\n",
453
				name, global.confdir);
Bernhard Link's avatar
Bernhard Link committed
454 455
		return RET_ERROR_MISSING;
	}
456 457

	r = incoming_prepare(imports.i);
458
	if (RET_WAS_ERROR(r)) {
459 460 461 462 463 464 465 466 467 468 469
		incoming_free(imports.i);
		return r;
	}
	*result = imports.i;
	return r;
}

struct candidate {
	/* from candidate_read */
	int ofs;
	char *control;
470
	struct signatures *signatures;
471
	/* from candidate_parse */
Bernhard Link's avatar
Bernhard Link committed
472
	char *source, *sourceversion, *changesversion;
473 474 475
	struct strlist distributions,
		       architectures,
		       binaries;
476
	bool isbinNMU;
477 478 479 480
	struct candidate_file {
		/* set by _addfileline */
		struct candidate_file *next;
		int ofs; /* to basename in struct incoming->files */
481 482 483
		filetype type;
		/* all NULL if it is the .changes itself,
		 * otherwise the data from the .changes for this file: */
484 485
		char *section;
		char *priority;
486
		architecture_t architecture;
487
		char *name;
488 489
		/* like above, but updated once files are copied */
		struct checksums *checksums;
490
		/* set later */
491
		bool used;
492
		char *tempfilename;
493 494
		/* distribution-unspecific contents of the packages */
		/* - only for FE_BINARY types: */
495
		struct deb_headers deb;
496
		/* - only for fe_DSC types */
497
		struct dsc_headers dsc;
498 499
		/* only valid while parsing */
		struct hashes h;
500
	} *files;
501 502 503
	struct candidate_perdistribution {
		struct candidate_perdistribution *next;
		struct distribution *into;
504
		bool skip;
505 506 507 508 509 510
		struct candidate_package {
			/* a package is something installing files, including
			 * the pseudo-package for the .changes file, if that is
			 * to be included */
			struct candidate_package *next;
			const struct candidate_file *master;
511
			component_t component;
512
			packagetype_t packagetype;
513 514 515 516 517 518 519 520
			struct strlist filekeys;
			/* a list of pointers to the files belonging to those
			 * filekeys, NULL if it does not need linking/copying */
			const struct candidate_file **files;
			/* only for FE_PACKAGE: */
			char *control;
			/* only for fe_DSC */
			char *directory;
521
			/* true if skipped because already there or newer */
522
			bool skip;
523
		} *packages;
524 525 526 527 528
		struct byhandfile {
			struct byhandfile *next;
			const struct candidate_file *file;
			const struct byhandhook *hook;
		} *byhandhookstocall;
529
	} *perdistribution;
530 531
	/* the logsubdir, and the list of files to put there,
	 * otherwise both NULL */
532 533 534
	char *logsubdir;
	int logcount;
	const struct candidate_file **logfiles;
535 536
};

537
static void candidate_file_free(/*@only@*/struct candidate_file *f) {
538
	checksums_free(f->checksums);
539 540 541
	free(f->section);
	free(f->priority);
	free(f->name);
542
	if (FE_BINARY(f->type))
543
		binaries_debdone(&f->deb);
544
	if (f->type == fe_DSC)
545
		sources_done(&f->dsc);
546
	if (f->tempfilename != NULL) {
547
		(void)unlink(f->tempfilename);
548 549 550 551 552 553
		free(f->tempfilename);
		f->tempfilename = NULL;
	}
	free(f);
}

554
static void candidate_package_free(/*@only@*/struct candidate_package *p) {
555 556 557 558 559 560 561
	free(p->control);
	free(p->directory);
	strlist_done(&p->filekeys);
	free(p->files);
	free(p);
}

562
static void candidate_free(/*@only@*/struct candidate *c) {
563
	if (c == NULL)
564 565
		return;
	free(c->control);
566
	signatures_free(c->signatures);
567
	free(c->source);
Bernhard Link's avatar
Bernhard Link committed
568 569
	free(c->sourceversion);
	free(c->changesversion);
570 571 572
	strlist_done(&c->distributions);
	strlist_done(&c->architectures);
	strlist_done(&c->binaries);
573
	while (c->perdistribution != NULL) {
574 575 576
		struct candidate_perdistribution *d = c->perdistribution;
		c->perdistribution = d->next;

577
		while (d->packages != NULL) {
578 579 580 581
			struct candidate_package *p = d->packages;
			d->packages = p->next;
			candidate_package_free(p);
		}
582
		while (d->byhandhookstocall != NULL) {
583 584 585 586
			struct byhandfile *h = d->byhandhookstocall;
			d->byhandhookstocall = h->next;
			free(h);
		}
587 588
		free(d);
	}
589
	while (c->files != NULL) {
590 591
		struct candidate_file *f = c->files;
		c->files = f->next;
592
		candidate_file_free(f);
593
	}
594 595
	free(c->logsubdir);
	free(c->logfiles);
596 597 598
	free(c);
}

599
static retvalue candidate_newdistribution(struct candidate *c, struct distribution *distribution) {
600
	struct candidate_perdistribution *n, **pp = &c->perdistribution;
601

602 603
	while (*pp != NULL) {
		if ((*pp)->into == distribution)
604 605 606
			return RET_NOTHING;
		pp = &(*pp)->next;
	}
607 608
	n = zNEW(struct candidate_perdistribution);
	if (FAILEDTOALLOC(n))
609 610 611 612 613 614 615
		return RET_ERROR_OOM;
	n->into = distribution;
	*pp = n;
	return RET_OK;
}

static struct candidate_package *candidate_newpackage(struct candidate_perdistribution *fordistribution, const struct candidate_file *master) {
616
	struct candidate_package *n, **pp = &fordistribution->packages;
617

618
	while (*pp != NULL)
619
		pp = &(*pp)->next;
620 621 622 623 624 625 626
	n = zNEW(struct candidate_package);
	if (FAILEDTOALLOC(n))
		return NULL;
	n->component = atom_unknown;
	n->packagetype = atom_unknown;
	n->master = master;
	*pp = n;
627 628 629
	return n;
}

630
static retvalue candidate_usefile(const struct incoming *i, const struct candidate *c, struct candidate_file *file);
631

632
static retvalue candidate_read(struct incoming *i, int ofs, struct candidate **result, bool *broken) {
633 634 635
	struct candidate *n;
	retvalue r;

636 637
	n = zNEW(struct candidate);
	if (FAILEDTOALLOC(n))
638 639
		return RET_ERROR_OOM;
	n->ofs = ofs;
640
	/* first file of any .changes file is the file itself */
641 642
	n->files = zNEW(struct candidate_file);
	if (FAILEDTOALLOC(n->files)) {
643 644 645
		free(n);
		return RET_ERROR_OOM;
	}
646
	n->files->ofs = n->ofs;
647
	n->files->type = fe_CHANGES;
648
	r = candidate_usefile(i, n, n->files);
649 650
	assert (r != RET_NOTHING);
	if (RET_WAS_ERROR(r)) {
651
		candidate_free(n);
652 653
		return r;
	}
654 655 656 657 658
	assert (n->files->tempfilename != NULL);
	r = signature_readsignedchunk(n->files->tempfilename, BASENAME(i, ofs),
			&n->control, &n->signatures, broken);
	assert (r != RET_NOTHING);
	if (RET_WAS_ERROR(r)) {
659
		candidate_free(n);
660
		return r;
661
	}
662 663 664 665 666 667
	*result = n;
	return RET_OK;
}

static retvalue candidate_addfileline(struct incoming *i, struct candidate *c, const char *fileline) {
	struct candidate_file **p, *n;
668
	char *basefilename;
669
	retvalue r;
670

671 672
	n = zNEW(struct candidate_file);
	if (FAILEDTOALLOC(n))
673 674
		return RET_ERROR_OOM;

675
	r = changes_parsefileline(fileline, &n->type, &basefilename,
676
			&n->h.hashes[cs_md5sum], &n->h.hashes[cs_length],
677 678 679
			&n->section, &n->priority, &n->architecture,
			&n->name);
	if (RET_WAS_ERROR(r)) {
680 681 682
		free(n);
		return r;
	}
683
	n->ofs = strlist_ofs(&i->files, basefilename);
684
	if (n->ofs < 0) {
685 686 687 688
		fprintf(stderr,
"In '%s': file '%s' not found in the incoming dir!\n",
				i->files.values[c->ofs], basefilename);
		free(basefilename);
689
		candidate_file_free(n);
690 691
		return RET_ERROR_MISSING;
	}
692
	free(basefilename);
693 694

	p = &c->files;
695
	while (*p != NULL)
696 697 698 699 700
		p = &(*p)->next;
	*p = n;
	return RET_OK;
}

701 702 703
static retvalue candidate_addhashes(struct incoming *i, struct candidate *c, enum checksumtype cs, const struct strlist *lines) {
	int j;

704
	for (j = 0 ; j < lines->count ; j++) {
705 706
		const char *fileline = lines->values[j];
		struct candidate_file *f;
707
		const char *basefilename;
708 709 710 711
		struct hash_data hash, size;
		retvalue r;

		r = hashline_parse(BASENAME(i, c->ofs), fileline, cs,
712
				&basefilename, &hash, &size);
713
		if (!RET_IS_OK(r))
714 715
			return r;
		f = c->files;
716
		while (f != NULL && strcmp(BASENAME(i, f->ofs), basefilename) != 0)
717
			f = f->next;
718
		if (f == NULL) {
719 720
			fprintf(stderr,
"Warning: Ignoring file '%s' listed in '%s' but not in '%s' of '%s'!\n",
721
					basefilename, changes_checksum_names[cs],
722 723 724 725
					changes_checksum_names[cs_md5sum],
					BASENAME(i, c->ofs));
			continue;
		}
726
		if (f->h.hashes[cs_length].len != size.len ||
727
				memcmp(f->h.hashes[cs_length].start,
728
					size.start, size.len) != 0) {
729 730
			fprintf(stderr,
"Error: Different size of '%s' listed in '%s' and '%s' of '%s'!\n",
731
					basefilename, changes_checksum_names[cs],
732 733 734 735 736 737 738 739 740
					changes_checksum_names[cs_md5sum],
					BASENAME(i, c->ofs));
			return RET_ERROR;
		}
		f->h.hashes[cs] = hash;
	}
	return RET_OK;
}

741
static retvalue candidate_finalizechecksums(struct candidate *c) {
742 743 744 745 746
	struct candidate_file *f;
	retvalue r;

	/* store collected hashes as checksums structs,
	 * starting after .changes file: */
747
	for (f = c->files->next ; f != NULL ; f = f->next) {
748
		r = checksums_initialize(&f->checksums, f->h.hashes);
749
		if (RET_WAS_ERROR(r))
750 751 752 753 754
			return r;
	}
	return RET_OK;
}

755 756
static retvalue candidate_parse(struct incoming *i, struct candidate *c) {
	retvalue r;
757 758
	struct strlist filelines[cs_hashCOUNT];
	enum checksumtype cs;
759
	int j;
760 761 762 763 764
#define R if (RET_WAS_ERROR(r)) return r;
#define E(err, ...) { \
		if (r == RET_NOTHING) { \
			fprintf(stderr, "In '%s': " err "\n", \
					BASENAME(i, c->ofs), ## __VA_ARGS__); \
765
			r = RET_ERROR; \
766 767
		} \
		if (RET_WAS_ERROR(r)) return r; \
768
	}
769 770
	r = chunk_getnameandversion(c->control, "Source", &c->source,
			&c->sourceversion);
771 772 773
	E("Missing 'Source' field!");
	r = propersourcename(c->source);
	E("Malforce Source name!");
774
	if (c->sourceversion != NULL) {
Bernhard Link's avatar
Bernhard Link committed
775 776 777
		r = properversion(c->sourceversion);
		E("Malforce Source Version number!");
	}
778
	r = chunk_getwordlist(c->control, "Architecture", &c->architectures);
Bernhard Link's avatar
Bernhard Link committed
779
	E("Missing 'Architecture' field!");
780 781 782 783 784
	r = chunk_getwordlist(c->control, "Binary", &c->binaries);
	if (r == RET_NOTHING)
		strlist_init(&c->binaries);
	else if (RET_WAS_ERROR(r))
		return r;
785
	r = chunk_getvalue(c->control, "Version", &c->changesversion);
786
	E("Missing 'Version' field!");
Bernhard Link's avatar
Bernhard Link committed
787
	r = properversion(c->changesversion);
788
	E("Malforce Version number!");
Bernhard Link's avatar
Bernhard Link committed
789
	// TODO: logic to detect binNMUs to warn against sources?
790
	if (c->sourceversion == NULL) {
Bernhard Link's avatar
Bernhard Link committed
791
		c->sourceversion = strdup(c->changesversion);
792
		if (FAILEDTOALLOC(c->sourceversion))
Bernhard Link's avatar
Bernhard Link committed
793
			return RET_ERROR_OOM;
794
		c->isbinNMU = false;
Bernhard Link's avatar
Bernhard Link committed
795 796 797 798 799 800 801
	} else {
		int cmp;

		r = dpkgversions_cmp(c->sourceversion, c->changesversion, &cmp);
		R;
		c->isbinNMU = cmp != 0;
	}
802
	r = chunk_getwordlist(c->control, "Distribution", &c->distributions);
803
	E("Missing 'Distribution' field!");
804 805 806 807
	r = chunk_getextralinelist(c->control,
			changes_checksum_names[cs_md5sum],
			&filelines[cs_md5sum]);
	E("Missing '%s' field!", changes_checksum_names[cs_md5sum]);
808
	for (j = 0 ; j < filelines[cs_md5sum].count ; j++) {
809
		r = candidate_addfileline(i, c, filelines[cs_md5sum].values[j]);
810
		if (RET_WAS_ERROR(r)) {
811 812 813 814
			strlist_done(&filelines[cs_md5sum]);
			return r;
		}
	}
815
	for (cs = cs_firstEXTENDED ; cs < cs_hashCOUNT ; cs++) {
816 817 818
		r = chunk_getextralinelist(c->control,
				changes_checksum_names[cs], &filelines[cs]);

819
		if (RET_IS_OK(r))
820 821 822 823
			r = candidate_addhashes(i, c, cs, &filelines[cs]);
		else
			strlist_init(&filelines[cs]);

824 825
		if (RET_WAS_ERROR(r)) {
			while (cs-- > cs_md5sum)
826
				strlist_done(&filelines[cs]);
827 828 829
			return r;
		}
	}
830
	r = candidate_finalizechecksums(c);
831
	for (cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++)
832 833
		strlist_done(&filelines[cs]);
	R;
834 835 836
	if (c->files == NULL || c->files->next == NULL) {
		fprintf(stderr, "In '%s': Empty 'Files' section!\n",
				BASENAME(i, c->ofs));
837 838 839 840
		return RET_ERROR;
	}
	return RET_OK;
}
841

842
static retvalue candidate_earlychecks(struct incoming *i, struct candidate *c) {
843 844 845 846 847 848
	struct candidate_file *file;
	retvalue r;

	// TODO: allow being more permissive,
	// that will need some more checks later, though
	r = propersourcename(c->source);
849
	if (RET_WAS_ERROR(r))
850
		return r;
Bernhard Link's avatar
Bernhard Link committed
851
	r = properversion(c->sourceversion);
852
	if (RET_WAS_ERROR(r))
853
		return r;
854 855
	for (file = c->files ; file != NULL ; file = file->next) {
		if (file->type != fe_CHANGES && file->type != fe_BYHAND &&
856
				file->type != fe_LOG &&
857
				!atom_defined(file->architecture)) {
858 859 860 861 862 863
			fprintf(stderr,
"'%s' contains '%s' not matching an valid architecture in any distribution known!\n",
					BASENAME(i, c->ofs),
					BASENAME(i, file->ofs));
			return RET_ERROR;
		}
864
		if (!FE_PACKAGE(file->type))
865
			continue;
866 867 868
		assert (atom_defined(file->architecture));
		if (strlist_in(&c->architectures,
					atoms_architectures[file->architecture]))
869
			continue;
870 871 872 873
		fprintf(stderr,
"'%s' is not listed in the Architecture header of '%s' but file '%s' looks like it!\n",
				atoms_architectures[file->architecture],
				BASENAME(i, c->ofs), BASENAME(i, file->ofs));
874 875
		return RET_ERROR;
	}
876 877
	return RET_OK;
}
878

879
/* Is used before any other candidate fields are set */
880
static retvalue candidate_usefile(const struct incoming *i, const struct candidate *c, struct candidate_file *file) {
881
	const char *basefilename;
882
	char *origfile, *tempfilename;
883
	struct checksums *readchecksums;
884
	retvalue r;
885
	bool improves;
Bernhard Link's avatar
Bernhard Link committed
886
	const char *p;
887

888
	if (file->used && file->tempfilename != NULL)
889
		return RET_OK;
890
	assert(file->tempfilename == NULL);
891
	basefilename = BASENAME(i, file->ofs);
892 893
	for (p = basefilename; *p != '\0' ; p++) {
		if ((0x80 & *(const unsigned char *)p) != 0) {
894 895 896
			fprintf(stderr,
"Invalid filename '%s' listed in '%s': contains 8-bit characters\n",
					basefilename, BASENAME(i, c->ofs));
Bernhard Link's avatar
Bernhard Link committed
897 898 899
			return RET_ERROR;
		}
	}
900
	tempfilename = calc_dirconcat(i->tempdir, basefilename);
901
	if (FAILEDTOALLOC(tempfilename))
902
		return RET_ERROR_OOM;
903
	origfile = calc_dirconcat(i->directory, basefilename);
904
	if (FAILEDTOALLOC(origfile)) {
905 906 907
		free(tempfilename);
		return RET_ERROR_OOM;
	}
908
	r = checksums_copyfile(tempfilename, origfile, true, &readchecksums);
909
	free(origfile);
910
	if (RET_WAS_ERROR(r)) {
911 912 913
		free(tempfilename);
		return r;
	}
914
	if (file->checksums == NULL) {
915 916 917 918 919
		file->checksums = readchecksums;
		file->tempfilename = tempfilename;
		file->used = true;
		return RET_OK;
	}
920
	if (!checksums_check(file->checksums, readchecksums, &improves)) {
921 922 923 924 925
		fprintf(stderr,
"ERROR: File '%s' does not match expectations:\n",
				basefilename);
		checksums_printdifferences(stderr,
				file->checksums, readchecksums);
926 927 928 929 930
		checksums_free(readchecksums);
		deletefile(tempfilename);
		free(tempfilename);
		return RET_ERROR_WRONG_MD5;
	}
931
	if (improves) {
932
		r = checksums_combine(&file->checksums, readchecksums, NULL);
933
		if (RET_WAS_ERROR(r)) {
934
			checksums_free(readchecksums);
935 936 937 938
			deletefile(tempfilename);
			free(tempfilename);
			return r;
		}
939 940
	}
	checksums_free(readchecksums);
941
	file->tempfilename = tempfilename;
942
	file->used = true;
943 944 945
	return RET_OK;
}

946
static inline retvalue getsectionprioritycomponent(const struct incoming *i, const struct candidate *c, const struct distribution *into, const struct candidate_file *file, const char *name, const struct overridedata *oinfo, /*@out@*/const char **section_p, /*@out@*/const char **priority_p, /*@out@*/component_t *component) {
947
	retvalue r;
948 949
	const char *section, *priority, *forcecomponent;
	component_t fc;
950 951

	section = override_get(oinfo, SECTION_FIELDNAME);
952
	if (section == NULL) {
953 954 955
		// TODO: warn about disparities here?
		section = file->section;
	}
956
	if (section == NULL || strcmp(section, "-") == 0) {
957 958
		fprintf(stderr, "No section found for '%s' ('%s' in '%s')!\n",
				name,
959
				BASENAME(i, file->ofs), BASENAME(i, c->ofs));
960 961
		return RET_ERROR;
	}
962
	priority = override_get(oinfo, PRIORITY_FIELDNAME);
963
	if (priority == NULL) {
964 965 966
		// TODO: warn about disparities here?
		priority = file->priority;
	}
967
	if (priority == NULL || strcmp(priority, "-") == 0) {
968 969
		fprintf(stderr, "No priority found for '%s' ('%s' in '%s')!\n",
				name,
970
				BASENAME(i, file->ofs), BASENAME(i, c->ofs));
971 972 973
		return RET_ERROR;
	}

974
	forcecomponent = override_get(oinfo, "$Component");
975
	if (forcecomponent != NULL) {
976
		fc = component_find(forcecomponent);
977
		if (!atom_defined(fc)) {
978 979 980 981 982 983 984 985 986
			fprintf(stderr,
"Unknown component '%s' (in $Component in override file for '%s'\n",
					forcecomponent, name);
			return RET_ERROR;
		}
		/* guess_component will check if that is valid for this
		 * distribution */
	} else
		fc = atom_unknown;
987
	r = guess_component(into->codename, &into->components,
988
			BASENAME(i, file->ofs), section,
989
			fc, component);
990
	if (RET_WAS_ERROR(r)) {
991 992
		return r;
	}
993 994 995 996 997
	*section_p = section;
	*priority_p = priority;
	return RET_OK;
}

998
static retvalue candidate_read_deb(struct incoming *i, struct candidate *c, struct candidate_file *file) {
999
	retvalue r;
Bernhard Link's avatar
Bernhard Link committed
1000 1001 1002
	size_t l;
	char *base;
	const char *packagenametocheck;
1003

1004
	r = binaries_readdeb(&file->deb, file->tempfilename);
1005
	if (RET_WAS_ERROR(r))
1006
		return r;
1007
	if (strcmp(file->name, file->deb.name) != 0) {
1008
		// TODO: add permissive thing to ignore this
1009 1010
		fprintf(stderr,
"Name part of filename ('%s') and name within the file ('%s') do not match for '%s' in '%s'!\n",
1011
				file->name, file->deb.name,
1012
				BASENAME(i, file->ofs), BASENAME(i, c->ofs));
1013 1014
		return RET_ERROR;
	}
1015
	if (file->architecture != file->deb.architecture) {
1016
		// TODO: add permissive thing to ignore this in some cases
1017
		// but do not forget to look into into->architectures then
1018 1019 1020 1021 1022 1023
		fprintf(stderr,
"Architecture '%s' of '%s' does not match '%s' specified in '%s'!\n",
				atoms_architectures[file->deb.architecture],
				BASENAME(i, file->ofs),
				atoms_architectures[file->architecture],
				BASENAME(i, c->ofs));
1024 1025
		return RET_ERROR;
	}
1026
	if (strcmp(c->source, file->deb.source) != 0) {
1027 1028
		// TODO: add permissive thing to ignore this
		// (beware if tracking is active)
1029 1030 1031 1032
		fprintf(stderr,
"Source header '%s' of '%s' and source name '%s' within the file '%s' do not match!\n",
				c->source, BASENAME(i, c->ofs),
				file->deb.source, BASENAME(i, file->ofs));
1033 1034
		return RET_ERROR;
	}
1035
	if (strcmp(c->sourceversion, file->deb.sourceversion) != 0) {
1036 1037
		// TODO: add permissive thing to ignore this
		// (beware if tracking is active)
1038 1039 1040 1041
		fprintf(stderr,
"Source version '%s' of '%s' and source version '%s' within the file '%s' do not match!\n",
				c->sourceversion, BASENAME(i, c->ofs),
				file->deb.sourceversion, BASENAME(i, file->ofs));
1042 1043
		return RET_ERROR;
	}
Bernhard Link's avatar
Bernhard Link committed
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057

	packagenametocheck = file->deb.name;
	l = strlen(file->deb.name);
	if (l > sizeof("-dbgsym")-1 &&
	    strcmp(file->deb.name + l - (sizeof("dbgsym")), "-dbgsym") == 0) {
		base = strndup(file->deb.name, l - (sizeof("dbgsym")));
		if (FAILEDTOALLOC(base))
			return RET_ERROR_OOM;
		packagenametocheck = base;
	} else {
		base = NULL;
	}

	if (! strlist_in(&c->binaries, packagenametocheck)
1058
	    && !i->permit[pmf_unlistedbinaries]) {
1059
		fprintf(stderr,
1060 1061
"Name '%s' of binary '%s' is not listed in Binaries header of '%s'!\n"
"(use Permit: unlisted_binaries in conf/incoming to ignore this error)\n",
Bernhard Link's avatar
Bernhard Link committed
1062
				packagenametocheck, BASENAME(i, file->ofs),
1063
				BASENAME(i, c->ofs));
Bernhard Link's avatar
Bernhard Link committed
1064
		free(base);
1065 1066
		return RET_ERROR;
	}
Bernhard Link's avatar
Bernhard Link committed
1067
	free(base);
1068
	r = properpackagename(file->deb.name);
1069
	if (RET_IS_OK(r))
1070
		r = propersourcename(file->deb.source);
1071
	if (RET_IS_OK(r))
1072
		r = properversion(file->deb.version);
1073
	if (RET_WAS_ERROR(r))
1074
		return r;
1075 1076 1077
	return RET_OK;
}

1078
static retvalue candidate_read_dsc(struct incoming *i, struct candidate_file *file) {
1079
	retvalue r;
1080
	bool broken = false;
1081 1082
	char *p;

1083 1084
	r = sources_readdsc(&file->dsc, file->tempfilename,
			BASENAME(i, file->ofs), &broken);
1085
	if (RET_WAS_ERROR(r))
1086 1087 1088
		return r;
	p = calc_source_basename(file->dsc.name,
			file->dsc.version);
1089
	if (FAILEDTOALLOC(p))
1090
		return RET_ERROR_OOM;
1091
	r = checksumsarray_include(&file->dsc.files, p, file->checksums);
1092
	if (RET_WAS_ERROR(r)) {
1093
		return r;
1094
	}
1095 1096 1097 1098 1099 1100 1101 1102
	// TODO: take a look at "broken"...
	return RET_OK;
}

static retvalue candidate_read_files(struct incoming *i, struct candidate *c) {
	struct candidate_file *file;
	retvalue r;

1103
	for (file = c->files ; file != NULL ; file = file->next) {
1104

1105
		if (!FE_PACKAGE(file->type))
1106 1107
			continue;
		r = candidate_usefile(i, c, file);
1108
		if (RET_WAS_ERROR(r))
1109 1110 1111
			return r;
		assert(file->tempfilename != NULL);

1112
		if (FE_BINARY(file->type))
1113
			r = candidate_read_deb(i, c, file);
1114
		else if (file->type == fe_DSC)
1115
			r = candidate_read_dsc(i, file);
1116 1117
		else {
			r = RET_ERROR;
1118
			assert (FE_BINARY(file->type) || file->type == fe_DSC);
1119
		}
1120
		if (RET_WAS_ERROR(r))
1121 1122 1123 1124 1125
			return r;
	}
	return RET_OK;
}

1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173