database.c 59.9 KB
Newer Older
1
/*  This file is part of "reprepro"
2
 *  Copyright (C) 2007,2008,2016 Bernhard R. Link
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 2 as
 *  published by the Free Software Foundation.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1301  USA
 */
#include <config.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <assert.h>
22
#include <limits.h>
23 24
#include <string.h>
#include <stdio.h>
25
#include <stdint.h>
26
#include <stdlib.h>
27 28
#include <fcntl.h>
#include <unistd.h>
29
#include <db.h>
30 31 32

#include "globals.h"
#include "error.h"
33
#include "ignore.h"
34 35 36 37
#include "strlist.h"
#include "names.h"
#include "database.h"
#include "dirs.h"
38
#include "filecntl.h"
39
#include "files.h"
40
#include "filelist.h"
41
#include "reference.h"
42
#include "tracking.h"
43
#include "dpkgversions.h"
44
#include "distribution.h"
45 46
#include "database_p.h"

47 48 49
#define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x)
#define LIBDB_VERSION_STRING "bdb" TOSTRING(DB_VERSION_MAJOR) "." TOSTRING(DB_VERSION_MINOR) "." TOSTRING(DB_VERSION_PATCH)
50 51 52
#define CLEARDBT(dbt) { memset(&dbt, 0, sizeof(dbt)); }
#define SETDBT(dbt, datastr) {const char *my = datastr; memset(&dbt, 0, sizeof(dbt)); dbt.data = (void *)my; dbt.size = strlen(my) + 1;}
#define SETDBTl(dbt, datastr, datasize) {const char *my = datastr; memset(&dbt, 0, sizeof(dbt)); dbt.data = (void *)my; dbt.size = datasize;}
53

54 55 56 57 58 59 60 61 62 63
static bool rdb_initialized, rdb_used, rdb_locked, rdb_verbose;
static int rdb_dircreationdepth;
static bool rdb_nopackages, rdb_readonly;
static bool rdb_packagesdatabaseopen;
static bool rdb_trackingdatabaseopen;
static /*@null@*/ char *rdb_version, *rdb_lastsupportedversion,
	*rdb_dbversion, *rdb_lastsupporteddbversion;

struct table *rdb_checksums, *rdb_contents;
struct table *rdb_references;
64 65 66
static struct {
	bool createnewtables;
} rdb_capabilities;
67 68 69

static void database_free(void) {
	if (!rdb_initialized)
70
		return;
71 72 73 74 75 76 77 78 79
	free(rdb_version);
	rdb_version = NULL;
	free(rdb_lastsupportedversion);
	rdb_lastsupportedversion = NULL;
	free(rdb_dbversion);
	rdb_dbversion = NULL;
	free(rdb_lastsupporteddbversion);
	rdb_lastsupporteddbversion = NULL;
	rdb_initialized = false;
80 81
}

82 83 84 85
static inline char *dbfilename(const char *filename) {
	return calc_dirconcat(global.dbdir, filename);
}

86 87 88 89
/**********************/
/* lock file handling */
/**********************/

90
static retvalue database_lock(size_t waitforlock) {
91 92 93 94 95
	char *lockfile;
	int fd;
	retvalue r;
	size_t tries = 0;

96 97 98
	assert (!rdb_locked);
	rdb_dircreationdepth = 0;
	r = dir_create_needed(global.dbdir, &rdb_dircreationdepth);
99
	if (RET_WAS_ERROR(r))
100 101
		return r;

102
	lockfile = dbfilename("lockfile");
103
	if (FAILEDTOALLOC(lockfile))
104
		return RET_ERROR_OOM;
105 106 107
	fd = open(lockfile, O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW|O_NOCTTY,
			S_IRUSR|S_IWUSR);
	while (fd < 0) {
108
		int e = errno;
109 110
		if (e == EEXIST) {
			if (tries < waitforlock && ! interrupted()) {
111
				unsigned int timetosleep = 10;
112 113
				if (verbose >= 0)
					printf(
114
"Could not acquire lock: %s already exists!\nWaiting 10 seconds before trying again.\n",
115 116
						lockfile);
				while (timetosleep > 0)
117
					timetosleep = sleep(timetosleep);
118
				tries++;
119 120 121
				fd = open(lockfile, O_WRONLY|O_CREAT|O_EXCL
						|O_NOFOLLOW|O_NOCTTY,
						S_IRUSR|S_IWUSR);
122 123 124 125
				continue;

			}
			fprintf(stderr,
Bernhard Link's avatar
Bernhard Link committed
126
"The lock file '%s' already exists. There might be another instance with the\n"
127
"same database dir running. To avoid locking overhead, only one process\n"
Bernhard Link's avatar
Bernhard Link committed
128
"can access the database at the same time. Do not delete the lock file unless\n"
129 130 131
"you are sure no other version is still running!\n", lockfile);

		} else
132 133
			fprintf(stderr,
"Error %d creating lock file '%s': %s!\n",
Bernhard Link's avatar
Bernhard Link committed
134
					e, lockfile, strerror(e));
135 136 137 138 139
		free(lockfile);
		return RET_ERRNO(e);
	}
	// TODO: do some more locking of this file to avoid problems
	// with the non-atomity of O_EXCL with nfs-filesystems...
140
	if (close(fd) != 0) {
141
		int e = errno;
142 143
		fprintf(stderr,
"(Late) Error %d creating lock file '%s': %s!\n",
Bernhard Link's avatar
Bernhard Link committed
144
				e, lockfile, strerror(e));
145 146 147 148 149
		(void)unlink(lockfile);
		free(lockfile);
		return RET_ERRNO(e);
	}
	free(lockfile);
150
	rdb_locked = true;
151 152 153
	return RET_OK;
}

154
static void releaselock(void) {
155 156
	char *lockfile;

157
	assert (rdb_locked);
158

159
	lockfile = dbfilename("lockfile");
160
	if (lockfile == NULL)
161
		return;
162
	if (unlink(lockfile) != 0) {
163
		int e = errno;
Bernhard Link's avatar
Bernhard Link committed
164 165
		fprintf(stderr, "Error %d deleting lock file '%s': %s!\n",
				e, lockfile, strerror(e));
166 167 168
		(void)unlink(lockfile);
	}
	free(lockfile);
169 170
	dir_remove_new(global.dbdir, rdb_dircreationdepth);
	rdb_locked = false;
171 172
}

173
static retvalue writeversionfile(void);
174

175
retvalue database_close(void) {
176
	retvalue result = RET_OK, r;
177

178 179
	if (rdb_references != NULL) {
		r = table_close(rdb_references);
180
		RET_UPDATE(result, r);
181
		rdb_references = NULL;
182
	}
183 184
	if (rdb_checksums != NULL) {
		r = table_close(rdb_checksums);
185
		RET_UPDATE(result, r);
186
		rdb_checksums = NULL;
187
	}
188 189
	if (rdb_contents != NULL) {
		r = table_close(rdb_contents);
190
		RET_UPDATE(result, r);
191
		rdb_contents = NULL;
192
	}
193
	r = writeversionfile();
194
	RET_UPDATE(result, r);
195 196 197
	if (rdb_locked)
		releaselock();
	database_free();
198
	return result;
199
}
200

201
static retvalue database_hasdatabasefile(const char *filename, /*@out@*/bool *exists_p) {
202 203
	char *fullfilename;

204
	fullfilename = dbfilename(filename);
205
	if (FAILEDTOALLOC(fullfilename))
206 207 208 209 210 211 212 213 214 215 216 217
		return RET_ERROR_OOM;
	*exists_p = isregularfile(fullfilename);
	free(fullfilename);
	return RET_OK;
}

enum database_type {
	dbt_QUERY,
	dbt_BTREE, dbt_BTREEDUP, dbt_BTREEPAIRS,
	dbt_HASH,
	dbt_COUNT /* must be last */
};
Bernhard Link's avatar
Bernhard Link committed
218
static const uint32_t types[dbt_COUNT] = {
219 220 221 222 223
	DB_UNKNOWN,
	DB_BTREE, DB_BTREE, DB_BTREE,
	DB_HASH
};

224 225 226
#if DB_VERSION_MAJOR >= 6
static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b, size_t *locp);
#else
227
static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b);
228
#endif
229

230
static retvalue database_opentable(const char *filename, /*@null@*/const char *subtable, enum database_type type, uint32_t flags, /*@out@*/DB **result) {
231 232 233 234
	char *fullfilename;
	DB *table;
	int dbret;

235
	fullfilename = dbfilename(filename);
236
	if (FAILEDTOALLOC(fullfilename))
237 238 239
		return RET_ERROR_OOM;

	dbret = db_create(&table, NULL, 0);
240
	if (dbret != 0) {
241 242 243 244
		fprintf(stderr, "db_create: %s\n", db_strerror(dbret));
		free(fullfilename);
		return RET_DBERR(dbret);
	}
245
	if (type == dbt_BTREEDUP || type == dbt_BTREEPAIRS) {
246
		dbret = table->set_flags(table, DB_DUPSORT);
247
		if (dbret != 0) {
248
			table->err(table, dbret, "db_set_flags(DB_DUPSORT):");
249 250 251 252 253
			(void)table->close(table, 0);
			free(fullfilename);
			return RET_DBERR(dbret);
		}
	}
254
	if (type == dbt_BTREEPAIRS) {
255
		dbret = table->set_dup_compare(table,  paireddatacompare);
256
		if (dbret != 0) {
257 258 259 260 261 262 263
			table->err(table, dbret, "db_set_dup_compare:");
			(void)table->close(table, 0);
			free(fullfilename);
			return RET_DBERR(dbret);
		}
	}

264
#if DB_VERSION_MAJOR == 5 || DB_VERSION_MAJOR == 6
265 266
#define DB_OPEN(database, filename, name, type, flags) \
	database->open(database, NULL, filename, name, type, flags, 0664)
Bernhard Link's avatar
Bernhard Link committed
267
#else
268
#if DB_VERSION_MAJOR == 4
269 270
#define DB_OPEN(database, filename, name, type, flags) \
	database->open(database, NULL, filename, name, type, flags, 0664)
271
#else
272
#if DB_VERSION_MAJOR == 3
273 274
#define DB_OPEN(database, filename, name, type, flags) \
	database->open(database, filename, name, type, flags, 0664)
275
#else
276
#error Unexpected DB_VERSION_MAJOR!
277
#endif
Bernhard Link's avatar
Bernhard Link committed
278
#endif
279
#endif
280
	dbret = DB_OPEN(table, fullfilename, subtable, types[type], flags);
281
	if (dbret == ENOENT && !ISSET(flags, DB_CREATE)) {
282 283 284 285
		(void)table->close(table, 0);
		free(fullfilename);
		return RET_NOTHING;
	}
286
	if (dbret != 0) {
287
		if (subtable != NULL)
288 289
			table->err(table, dbret, "db_open(%s:%s)[%d]",
					fullfilename, subtable, dbret);
290
		else
291 292
			table->err(table, dbret, "db_open(%s)[%d]",
					fullfilename, dbret);
293 294 295 296 297 298 299 300 301
		(void)table->close(table, 0);
		free(fullfilename);
		return RET_DBERR(dbret);
	}
	free(fullfilename);
	*result = table;
	return RET_OK;
}

302
retvalue database_listsubtables(const char *filename, struct strlist *result) {
303 304
	DB *table;
	DBC *cursor;
305
	DBT key, data;
306
	int dbret;
307
	retvalue ret, r;
308 309
	struct strlist ids;

310
	r = database_opentable(filename, NULL,
311
			dbt_QUERY, DB_RDONLY, &table);
312
	if (!RET_IS_OK(r))
313 314 315
		return r;

	cursor = NULL;
316
	if ((dbret = table->cursor(table, NULL, &cursor, 0)) != 0) {
317
		table->err(table, dbret, "cursor(%s):", filename);
318
		(void)table->close(table, 0);
319 320 321 322 323 324 325 326
		return RET_ERROR;
	}
	CLEARDBT(key);
	CLEARDBT(data);

	strlist_init(&ids);

	ret = RET_NOTHING;
327
	while ((dbret=cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) {
328
		char *identifier = strndup(key.data, key.size);
329
		if (FAILEDTOALLOC(identifier)) {
330 331 332 333 334
			(void)table->close(table, 0);
			strlist_done(&ids);
			return RET_ERROR_OOM;
		}
		r = strlist_add(&ids, identifier);
335
		if (RET_WAS_ERROR(r)) {
336 337 338 339
			(void)table->close(table, 0);
			strlist_done(&ids);
			return r;
		}
340
		ret = RET_OK;
341 342 343 344
		CLEARDBT(key);
		CLEARDBT(data);
	}

345
	if (dbret != 0 && dbret != DB_NOTFOUND) {
346 347 348 349 350
		table->err(table, dbret, "c_get(%s):", filename);
		(void)table->close(table, 0);
		strlist_done(&ids);
		return RET_DBERR(dbret);
	}
351
	if ((dbret = cursor->c_close(cursor)) != 0) {
352 353 354 355 356 357 358
		table->err(table, dbret, "c_close(%s):", filename);
		(void)table->close(table, 0);
		strlist_done(&ids);
		return RET_DBERR(dbret);
	}

	dbret = table->close(table, 0);
359
	if (dbret != 0) {
360 361 362 363 364
		table->err(table, dbret, "close(%s):", filename);
		strlist_done(&ids);
		return RET_DBERR(dbret);
	} else {
		strlist_move(result, &ids);
365
		return ret;
366 367
	}
}
368

369
retvalue database_dropsubtable(const char *table, const char *subtable) {
370 371 372 373
	char *filename;
	DB *db;
	int dbret;

374
	filename = dbfilename(table);
375
	if (FAILEDTOALLOC(filename))
376 377 378
		return RET_ERROR_OOM;

	if ((dbret = db_create(&db, NULL, 0)) != 0) {
379 380
		fprintf(stderr, "db_create: %s %s\n",
				filename, db_strerror(dbret));
381 382 383 384
		free(filename);
		return RET_DBERR(dbret);
	}
	dbret = db->remove(db, filename, subtable, 0);
385
	if (dbret == ENOENT) {
386 387 388 389
		free(filename);
		return RET_NOTHING;
	}
	if (dbret != 0) {
390
		fprintf(stderr, "Error removing '%s' from %s!\n",
391 392 393 394 395 396 397 398
				subtable, filename);
		free(filename);
		return RET_DBERR(dbret);
	}

	free(filename);
	return RET_OK;
}
399

400 401 402 403
static inline bool targetisdefined(const char *identifier, struct distribution *distributions) {
	struct distribution *d;
	struct target *t;

404 405 406
	for (d = distributions ; d != NULL ; d = d->next) {
		for (t = d->targets; t != NULL ; t = t->next) {
			if (strcmp(t->identifier, identifier) == 0) {
407
				t->existed = true;
408
				return true;
409
			}
410 411 412 413 414
		}
	}
	return false;
}

415
static retvalue warnidentifiers(const struct strlist *identifiers, struct distribution *distributions, bool readonly) {
416 417
	struct distribution *d;
	struct target *t;
418
	const char *identifier;
419
	retvalue r;
420 421
	int i;

422
	for (i = 0; i < identifiers->count ; i++) {
423 424
		identifier = identifiers->values[i];

425
		if (targetisdefined(identifier, distributions))
426 427 428 429
			continue;

		fprintf(stderr,
"Error: packages database contains unused '%s' database.\n", identifier);
430
		if (ignored[IGN_undefinedtarget] == 0) {
431
			(void)fputs(
432 433 434 435 436 437
"This usually means you removed some component, architecture or even\n"
"a whole distribution from conf/distributions.\n"
"In that case you most likely want to call reprepro clearvanished to get rid\n"
"of the databases belonging to those removed parts.\n"
"(Another reason to get this error is using conf/ and db/ directories\n"
" belonging to different reprepro repositories).\n",
438 439
					stderr);
		}
440 441 442
		if (IGNORABLE(undefinedtarget)) {
			(void)fputs(
"Ignoring as --ignore=undefinedtarget given.\n",
443 444 445 446 447
					stderr);
			ignored[IGN_undefinedtarget]++;
			continue;
		}

448 449 450
		(void)fputs(
"To ignore use --ignore=undefinedtarget.\n",
				stderr);
451 452
		return RET_ERROR;
	}
453
	if (readonly)
454
		return RET_OK;
455
	for (d = distributions ; d != NULL ; d = d->next) {
456
		bool architecture_existed[d->architectures.count];
457
		bool have_old = false;
458 459 460 461

		/* check for new architectures */
		memset(architecture_existed, 0, sizeof(architecture_existed));

462
		for (t = d->targets; t != NULL ; t = t->next) {
Bernhard Link's avatar
Bernhard Link committed
463
			int o;
464

465
			if (!t->existed)
466 467
				continue;

Bernhard Link's avatar
Bernhard Link committed
468
			o = atomlist_ofs(&d->architectures,
469 470 471
					t->architecture);
			assert (o >= 0);
			if (o >= 0) {
Bernhard Link's avatar
Bernhard Link committed
472
				architecture_existed[o] = true;
473 474 475 476 477
				/* only warn about new ones if there
				 * is at least one old one, otherwise
				 * it's just a new distribution */
				have_old = true;
			}
478
		}
479
		for (i = 0 ; have_old && i < d->architectures.count ; i++) {
480 481
			architecture_t a;

482
			if (architecture_existed[i])
483 484 485 486 487 488 489 490 491 492 493 494 495
				continue;

			a = d->architectures.atoms[i];

			fprintf(stderr,
"New architecture '%s' in '%s'. Perhaps you want to call\n"
"reprepro flood '%s' '%s'\n"
"to populate it with architecture 'all' packages from other architectures.\n",
				atoms_architectures[a], d->codename,
				d->codename, atoms_architectures[a]);
		}

		/* create databases, so we know next time what is new */
496 497
		for (t = d->targets; t != NULL ; t = t->next) {
			if (t->existed)
498 499 500 501
				continue;
			/* create database now, to test it can be created
			 * early, and to know when new architectures
			 * arrive in the future. */
502
			r = target_initpackagesdb(t, READWRITE);
503
			if (RET_WAS_ERROR(r))
504 505
				return r;
			r = target_closepackagesdb(t);
506
			if (RET_WAS_ERROR(r))
507 508 509
				return r;
		}
	}
510 511 512
	return RET_OK;
}

513 514 515 516 517
static retvalue warnunusedtracking(const struct strlist *codenames, const struct distribution *distributions) {
	const char *codename;
	const struct distribution *d;
	int i;

518
	for (i = 0; i < codenames->count ; i++) {
519 520 521
		codename = codenames->values[i];

		d = distributions;
522
		while (d != NULL && strcmp(d->codename, codename) != 0)
523
			d = d->next;
524
		if (d != NULL && d->tracking != dt_NONE)
525 526 527 528
			continue;

		fprintf(stderr,
"Error: tracking database contains unused '%s' database.\n", codename);
529 530
		if (ignored[IGN_undefinedtracking] == 0) {
			if (d == NULL)
531
				(void)fputs(
532
"This either means you removed a distribution from the distributions config\n"
Bernhard Link's avatar
Bernhard Link committed
533
"file without calling clearvanished (or at least removealltracks), you\n"
Bernhard Link's avatar
Bernhard Link committed
534
"experienced a bug in retrack in versions < 3.0.0, you found a new bug or your\n"
535 536 537
"config does not belong to this database.\n",
						stderr);
			else
538
				(void)fputs(
539 540 541 542
"This either means you removed the Tracking: options from this distribution without\n"
"calling removealltracks for it, or your config does not belong to this database.\n",
						stderr);
		}
543 544 545
		if (IGNORABLE(undefinedtracking)) {
			(void)fputs(
"Ignoring as --ignore=undefinedtracking given.\n",
546 547 548 549 550
					stderr);
			ignored[IGN_undefinedtracking]++;
			continue;
		}

551 552
		(void)fputs("To ignore use --ignore=undefinedtracking.\n",
				stderr);
553 554 555 556 557
		return RET_ERROR;
	}
	return RET_OK;
}

558
static retvalue readline(/*@out@*/char **result, FILE *f, const char *versionfilename) {
559
	char buffer[21];
560 561
	size_t l;

562
	if (fgets(buffer, 20, f) == NULL) {
563
		int e = errno;
564 565 566
		if (e == 0) {
			fprintf(stderr,
"Error reading '%s': unexpected empty file\n",
567 568 569 570 571 572 573 574 575
					versionfilename);
			return RET_ERROR;
		} else {
			fprintf(stderr, "Error reading '%s': %s(errno is %d)\n",
					versionfilename, strerror(e), e);
			return RET_ERRNO(e);
		}
	}
	l = strlen(buffer);
576
	while (l > 0 && (buffer[l-1] == '\r' || buffer[l-1] == '\n')) {
577 578
		buffer[--l] = '\0';
	}
579
	if (l == 0) {
580
		fprintf(stderr, "Error reading '%s': unexpected empty line.\n",
581 582 583
				versionfilename);
		return RET_ERROR;
	}
584
	*result = strdup(buffer);
585
	if (FAILEDTOALLOC(*result))
586 587 588
		return RET_ERROR_OOM;
	return RET_OK;
}
589

590
static retvalue readversionfile(bool nopackagesyet) {
591 592 593 594
	char *versionfilename;
	FILE *f;
	retvalue r;
	int c;
595

596
	versionfilename = dbfilename("version");
597
	if (FAILEDTOALLOC(versionfilename))
598 599
		return RET_ERROR_OOM;
	f = fopen(versionfilename, "r");
600
	if (f == NULL) {
601
		int e = errno;
602

603
		if (e != ENOENT) {
604
			fprintf(stderr, "Error opening '%s': %s(errno is %d)\n",
605 606 607 608
					versionfilename, strerror(e), e);
			free(versionfilename);
			return RET_ERRNO(e);
		}
609
		free(versionfilename);
610
		if (nopackagesyet) {
611
			/* set to default for new packages.db files: */
612 613
			rdb_version = strdup(VERSION);
			if (FAILEDTOALLOC(rdb_version))
614
				return RET_ERROR_OOM;
615
			rdb_capabilities.createnewtables = true;
616
		} else
617 618 619 620
			rdb_version = NULL;
		rdb_lastsupportedversion = NULL;
		rdb_dbversion = NULL;
		rdb_lastsupporteddbversion = NULL;
621
		return RET_NOTHING;
622
	}
623
	/* first line is the version creating this database */
624
	r = readline(&rdb_version, f, versionfilename);
625
	if (RET_WAS_ERROR(r)) {
626 627 628
		(void)fclose(f);
		free(versionfilename);
		return r;
629
	}
630 631
	/* second line says which versions of reprepro will be able to cope
	 * with this database */
632
	r = readline(&rdb_lastsupportedversion, f, versionfilename);
633
	if (RET_WAS_ERROR(r)) {
634 635
		(void)fclose(f);
		free(versionfilename);
636 637 638
		return r;
	}
	/* next line is the version of the underlying database library */
639
	r = readline(&rdb_dbversion, f, versionfilename);
640
	if (RET_WAS_ERROR(r)) {
641 642 643 644 645
		(void)fclose(f);
		free(versionfilename);
		return r;
	}
	/* and then the minimum version of this library needed. */
646
	r = readline(&rdb_lastsupporteddbversion, f, versionfilename);
647
	if (RET_WAS_ERROR(r)) {
648 649 650
		(void)fclose(f);
		free(versionfilename);
		return r;
651 652 653
	}
	(void)fclose(f);
	free(versionfilename);
654 655 656

	/* check for enabled capabilities in the version */

657
	r = dpkgversions_cmp(rdb_version, "3", &c);
658
	if (RET_WAS_ERROR(r))
659
		return r;
660
	if (c >= 0)
661
		rdb_capabilities.createnewtables = true;
662 663 664

	/* ensure we can understand it */

665
	r = dpkgversions_cmp(VERSION, rdb_lastsupportedversion, &c);
666
	if (RET_WAS_ERROR(r))
667
		return r;
668
	if (c < 0) {
669
		fprintf(stderr,
Bernhard Link's avatar
Bernhard Link committed
670 671
"According to %s/version this database was created with a future version\n"
"and uses features this version cannot understand. Aborting...\n",
672
				global.dbdir);
673 674 675 676 677
		return RET_ERROR;
	}

	/* ensure it's a libdb database: */

678
	if (strncmp(rdb_dbversion, "bdb", 3) != 0) {
679
		fprintf(stderr,
Bernhard Link's avatar
Bernhard Link committed
680
"According to %s/version this database was created with a yet unsupported\n"
681
"database library. Aborting...\n",
682
				global.dbdir);
683 684
		return RET_ERROR;
	}
685
	if (strncmp(rdb_lastsupporteddbversion, "bdb", 3) != 0) {
686
		fprintf(stderr,
Bernhard Link's avatar
Bernhard Link committed
687
"According to %s/version this database was created with a yet unsupported\n"
688
"database library. Aborting...\n",
689
				global.dbdir);
690 691
		return RET_ERROR;
	}
692 693
	r = dpkgversions_cmp(LIBDB_VERSION_STRING,
			rdb_lastsupporteddbversion, &c);
694
	if (RET_WAS_ERROR(r))
695
		return r;
696
	if (c < 0) {
697
		fprintf(stderr,
Bernhard Link's avatar
Bernhard Link committed
698 699 700
"According to %s/version this database was created with a future version\n"
"%s of libdb. The libdb version this binary is linked against cannot yet\n"
"handle this format. Aborting...\n",
701
				global.dbdir, rdb_dbversion + 3);
702 703
		return RET_ERROR;
	}
704 705 706
	return RET_OK;
}

707
static retvalue writeversionfile(void) {
708
	char *versionfilename, *finalversionfilename;
709
	FILE *f;
710
	int i, e;
711

712
	versionfilename = dbfilename("version.new");
713
	if (FAILEDTOALLOC(versionfilename))
714 715
		return RET_ERROR_OOM;
	f = fopen(versionfilename, "w");
716
	if (f == NULL) {
717
		e = errno;
718 719 720 721 722
		fprintf(stderr, "Error creating '%s': %s(errno is %d)\n",
					versionfilename, strerror(e), e);
		free(versionfilename);
		return RET_ERRNO(e);
	}
723
	if (rdb_version == NULL)
724
		(void)fputs("0\n", f);
725
	else {
726
		(void)fputs(rdb_version, f);
727
		(void)fputc('\n', f);
728
	}
729
	if (rdb_lastsupportedversion == NULL) {
730 731
		(void)fputs("3.3.0\n", f);
	} else {
732 733 734
		int c;
		retvalue r;

735 736
		r = dpkgversions_cmp(rdb_lastsupportedversion,
				"3.3.0", &c);
737
		if (!RET_IS_OK(r) || c < 0)
738 739
			(void)fputs("3.3.0\n", f);
		else {
740
			(void)fputs(rdb_lastsupportedversion, f);
741 742
			(void)fputc('\n', f);
		}
743
	}
744
	if (rdb_dbversion == NULL)
745 746
		fprintf(f, "bdb%d.%d.%d\n", DB_VERSION_MAJOR, DB_VERSION_MINOR,
				DB_VERSION_PATCH);
747
	else {
748
		(void)fputs(rdb_dbversion, f);
749
		(void)fputc('\n', f);
750
	}
751
	if (rdb_lastsupporteddbversion == NULL)
752 753
		fprintf(f, "bdb%d.%d.0\n", DB_VERSION_MAJOR, DB_VERSION_MINOR);
	else {
754
		(void)fputs(rdb_lastsupporteddbversion, f);
755
		(void)fputc('\n', f);
756
	}
757 758 759

	e = ferror(f);

760
	if (e != 0) {
761 762 763
		fprintf(stderr, "Error writing '%s': %s(errno is %d)\n",
				versionfilename, strerror(e), e);
		(void)fclose(f);
764
		unlink(versionfilename);
765 766 767
		free(versionfilename);
		return RET_ERRNO(e);
	}
768
	if (fclose(f) != 0) {
769 770 771
		e = errno;
		fprintf(stderr, "Error writing '%s': %s(errno is %d)\n",
				versionfilename, strerror(e), e);
772
		unlink(versionfilename);
773 774 775
		free(versionfilename);
		return RET_ERRNO(e);
	}
776
	finalversionfilename = dbfilename("version");
777
	if (FAILEDTOALLOC(finalversionfilename)) {
778 779 780 781 782 783
		unlink(versionfilename);
		free(versionfilename);
		return RET_ERROR_OOM;
	}

	i = rename(versionfilename, finalversionfilename);
784
	if (i != 0) {
785 786 787 788 789 790 791 792 793 794
		e = errno;
		fprintf(stderr, "Error %d moving '%s' to '%s': %s\n",
				e, versionfilename, finalversionfilename,
				strerror(e));
		(void)unlink(versionfilename);
		free(versionfilename);
		free(finalversionfilename);
		return RET_ERRNO(e);
	}
	free(finalversionfilename);
795 796 797 798
	free(versionfilename);
	return RET_OK;
}

799
static retvalue createnewdatabase(struct distribution *distributions) {
800 801 802 803
	struct distribution *d;
	struct target *t;
	retvalue result = RET_NOTHING, r;

804 805
	for (d = distributions ; d != NULL ; d = d->next) {
		for (t = d->targets ; t != NULL ; t = t->next) {
806
			r = target_initpackagesdb(t, READWRITE);
807
			RET_UPDATE(result, r);
808
			if (RET_IS_OK(r)) {
809 810 811 812 813
				r = target_closepackagesdb(t);
				RET_UPDATE(result, r);
			}
		}
	}
814
	r = writeversionfile();
815 816 817 818
	RET_UPDATE(result, r);
	return result;
}

819 820 821 822 823
/* Initialize a database.
 * - if not fast, make all kind of checks for consistency (TO BE IMPLEMENTED),
 * - if readonly, do not create but return with RET_NOTHING
 * - lock database, waiting a given amount of time if already locked
 */
824
retvalue database_create(struct distribution *alldistributions, bool fast, bool nopackages, bool allowunused, bool readonly, size_t waitforlock, bool verbosedb) {
825
	retvalue r;
826
	bool packagesfileexists, trackingfileexists, nopackagesyet;
827

828 829 830 831 832 833
	if (rdb_initialized || rdb_used) {
		fputs("Internal Error: database initialized a 2nd time!\n",
				stderr);
		return RET_ERROR_INTERNAL;
	}

834 835 836 837
	if (readonly && !isdir(global.dbdir)) {
		if (verbose >= 0)
			fprintf(stderr,
"Exiting without doing anything, as there is no database yet that could result in other actions.\n");
838 839 840
		return RET_NOTHING;
	}

841 842
	rdb_initialized = true;
	rdb_used = true;
843

844
	r = database_lock(waitforlock);
845 846
	assert (r != RET_NOTHING);
	if (!RET_IS_OK(r)) {
847
		database_free();
848 849
		return r;
	}
850 851
	rdb_readonly = readonly;
	rdb_verbose = verbosedb;
852

853 854 855 856
	r = database_hasdatabasefile("packages.db", &packagesfileexists);
	if (RET_WAS_ERROR(r)) {
		releaselock();
		database_free();
857
		return r;
858 859 860 861 862
	}
	r = database_hasdatabasefile("tracking.db", &trackingfileexists);
	if (RET_WAS_ERROR(r)) {
		releaselock();
		database_free();
863
		return r;
864
	}
865 866 867

	nopackagesyet = !packagesfileexists && !trackingfileexists;

868
	r = readversionfile(nopackagesyet);
869
	if (RET_WAS_ERROR(r)) {
870 871
		releaselock();
		database_free();
872 873
		return r;
	}
874

875
	if (nopackages) {
876
		rdb_nopackages = true;
877 878 879
		return RET_OK;
	}

880
	if (nopackagesyet) {
881 882 883
		// TODO: handle readonly, but only once packages files may no
		// longer be generated when it is active...

884
		r = createnewdatabase(alldistributions);
885
		if (RET_WAS_ERROR(r)) {
886
			database_close();
887 888 889 890
			return r;
		}
	}

891 892
	/* after this point we should call database_close,
	 * as other stuff was handled,
893 894
	 * so writing the version file cannot harm (and not doing so could) */

895
	if (!allowunused && !fast && packagesfileexists)  {
896 897
		struct strlist identifiers;

898
		r = database_listpackages(&identifiers);
899
		if (RET_WAS_ERROR(r)) {
900
			database_close();
901 902
			return r;
		}
903 904
		if (r == RET_NOTHING)
			strlist_init(&identifiers);
905
		r = warnidentifiers(&identifiers,
906 907
				alldistributions, readonly);
		if (RET_WAS_ERROR(r)) {
908
			strlist_done(&identifiers);
909 910
			database_close();
			return r;
911
		}
912
		strlist_done(&identifiers);
913
	}
914
	if (!allowunused && !fast && trackingfileexists)  {
915 916
		struct strlist codenames;

917
		r = tracking_listdistributions(&codenames);
918
		if (RET_WAS_ERROR(r)) {
919
			database_close();
920 921
			return r;
		}
922
		if (RET_IS_OK(r)) {
923
			r = warnunusedtracking(&codenames, alldistributions);
924
			if (RET_WAS_ERROR(r)) {
925
				strlist_done(&codenames);
926
				database_close();
927 928 929 930
				return r;
			}
			strlist_done(&codenames);
		}
931 932
	}

933 934 935
	return RET_OK;
}

936 937 938
/****************************************************************************
 * Stuff string parts                                                       *
 ****************************************************************************/
939

Bernhard Link's avatar
Bernhard Link committed
940
static const char databaseerror[] = "Internal error of the underlying BerkeleyDB database:\n";
941

942 943 944
/****************************************************************************
 * Stuff to handle data in tables                                           *
 ****************************************************************************
945
 There is nothing that cannot be solved by another layer of indirection, except
946 947 948 949
 too many levels of indirection. (Source forgotten) */

struct table {
	char *name, *subname;
950
	DB *berkeleydb;
951 952 953 954 955
	bool *flagreset;
	bool readonly, verbose;
};

static void table_printerror(struct table *table, int dbret, const char *action) {
956
	if (table->subname != NULL)
957
		table->berkeleydb->err(table->berkeleydb, dbret,
958
				"%sWithin %s subtable %s at %s",
959 960 961
				databaseerror, table->name, table->subname,
				action);
	else
962
		table->berkeleydb->err(table->berkeleydb, dbret,
963
				"%sWithin %s at %s",
964 965 966 967 968
				databaseerror, table->name, action);
}

retvalue table_close(struct table *table) {
	int dbret;
969
	retvalue result;
970

971
	if (table == NULL)
972
		return RET_NOTHING;
973
	if (table->flagreset != NULL)
974
		*table->flagreset = false;
975 976
	if (table->berkeleydb == NULL) {
		assert (table->readonly);
977 978
		dbret = 0;
	} else
979
		dbret = table->berkeleydb->close(table->berkeleydb, 0);
980
	if (dbret != 0) {
981 982 983 984 985 986
		fprintf(stderr, "db_close(%s, %s): %s\n",
				table->name, table->subname,
				db_strerror(dbret));
		result = RET_DBERR(dbret);
	} else
		result = RET_OK;
987 988 989
	free(table->name);
	free(table->subname);
	free(table);
990
	return result;
991 992
}

993
retvalue table_getrecord(struct table *table, const char *key, char **data_p, size_t *datalen_p) {
994 995 996
	int dbret;
	DBT Key, Data;

997 998 999
	assert (table != NULL);
	if (table->berkeleydb == NULL) {
		assert (table->readonly);
1000 1001 1002 1003 1004 1005 1006
		return RET_NOTHING;
	}

	SETDBT(Key, key);
	CLEARDBT(Data);
	Data.flags = DB_DBT_MALLOC;

1007
	dbret = table->berkeleydb->get(table->berkeleydb, NULL,
1008 1009
			&Key, &Data, 0);
	// TODO: find out what error code means out of memory...
1010
	if (dbret == DB_NOTFOUND)
1011
		return RET_NOTHING;
1012
	if (dbret != 0) {
1013 1014 1015
		table_printerror(table, dbret, "get");
		return RET_DBERR(dbret);
	}
1016
	if (FAILEDTOALLOC(Data.data))
1017
		return RET_ERROR_OOM;
1018 1019 1020
	if (Data.size <= 0 ||
	    ((const char*)Data.data)[Data.size-1] != '\0') {
		if (table->subname != NULL)
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
			fprintf(stderr,
"Database %s(%s) returned corrupted (not null-terminated) data!",
					table->name, table->subname);
		else
			fprintf(stderr,
"Database %s returned corrupted (not null-terminated) data!",
					table->name);
		free(Data.data);
		return RET_ERROR;
	}
	*data_p = Data.data;
1032 1033
	if (datalen_p != NULL)
		*datalen_p = Data.size-1;
1034 1035 1036
	return RET_OK;
}

1037 1038 1039 1040 1041
retvalue table_getpair(struct table *table, const char *key, const char *value, /*@out@*/const char **data_p, /*@out@*/size_t *datalen_p) {
	int dbret;
	DBT Key, Data;
	size_t valuelen = strlen(value);

1042 1043 1044
	assert (table != NULL);
	if (table->berkeleydb == NULL) {
		assert (table->readonly);
1045 1046 1047 1048 1049 1050
		return RET_NOTHING;
	}

	SETDBT(Key, key);
	SETDBTl(Data, value, valuelen + 1);

1051
	dbret = table->berkeleydb->get(table->berkeleydb, NULL,
1052
			&Key, &Data, DB_GET_BOTH);
1053
	if (dbret == DB_NOTFOUND || dbret == DB_KEYEMPTY)
1054
		return RET_NOTHING;
1055
	if (dbret != 0) {
1056 1057 1058
		table_printerror(table, dbret, "get(BOTH)");
		return RET_DBERR(dbret);
	}
1059
	if (FAILEDTOALLOC(Data.data))
1060
		return RET_ERROR_OOM;
1061 1062 1063
	if (Data.size < valuelen + 2  ||
	    ((const char*)Data.data)[Data.size-1] != '\0') {
		if (table->subname != NULL)
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
			fprintf(stderr,
"Database %s(%s) returned corrupted (not paired) data!",
					table->name, table->subname);
		else
			fprintf(stderr,
"Database %s returned corrupted (not paired) data!",
					table->name);
		return RET_ERROR;
	}
	*data_p = ((const char*)Data.data) + valuelen + 1;
	*datalen_p = Data.size - valuelen - 2;
	return RET_OK;
}

1078
retvalue table_gettemprecord(struct table *table, const char *key, const char **data_p, size_t *datalen_p) {
1079 1080 1081
	int dbret;
	DBT Key, Data;

1082 1083 1084
	assert (table != NULL);
	if (table->berkeleydb == NULL) {
		assert (table->readonly);
1085 1086 1087 1088 1089 1090
		return RET_NOTHING;
	}

	SETDBT(Key, key);
	CLEARDBT(Data);

1091
	dbret = table->berkeleydb->get(table->berkeleydb, NULL,
1092 1093
			&Key, &Data, 0);
	// TODO: find out what error code means out of memory...
1094
	if (dbret == DB_NOTFOUND)
1095
		return RET_NOTHING;
1096
	if (dbret != 0) {
1097 1098 1099
		table_printerror(table, dbret, "get");
		return RET_DBERR(dbret);
	}
1100
	if (FAILEDTOALLOC(Data.data))
1101
		return RET_ERROR_OOM;
1102 1103
	if (data_p == NULL) {
		assert (datalen_p == NULL);
1104 1105
		return RET_OK;
	}
1106 1107 1108
	if (Data.size <= 0 ||
	    ((const char*)Data.data)[Data.size-1] != '\0') {
		if (table->subname != NULL)
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
			fprintf(stderr,
"Database %s(%s) returned corrupted (not null-terminated) data!",
					table->name, table->subname);
		else
			fprintf(stderr,
"Database %s returned corrupted (not null-terminated) data!",
					table->name);
		return RET_ERROR;
	}
	*data_p = Data.data;
1119
	if (datalen_p != NULL)
1120
		*datalen_p = Data.size - 1;
1121 1122 1123
	return RET_OK;
}

1124
retvalue table_checkrecord(struct table *table, const char *key, const char *data) {
1125 1126
	int dbret;
	DBT Key, Data;
1127 1128
	DBC *cursor;
	retvalue r;
1129

1130 1131
	SETDBT(Key, key);
	SETDBT(Data, data);
1132
	dbret = table->berkeleydb->cursor(table->berkeleydb, NULL, &cursor, 0);
1133
	if (dbret != 0) {
1134 1135
		table_printerror(table, dbret, "cursor");
		return RET_DBERR(dbret);
1136
	}
1137
	dbret=cursor->c_get(cursor, &Key, &Data, DB_GET_BOTH);
1138
	if (dbret == 0) {
1139
		r = RET_OK;
1140
	} else if (dbret == DB_NOTFOUND) {
1141 1142 1143 1144 1145 1146 1147
		r = RET_NOTHING;
	} else {
		table_printerror(table, dbret, "c_get");
		(void)cursor->c_close(cursor);
		return RET_DBERR(dbret);
	}
	dbret = cursor->c_close(cursor);
1148
	if (dbret != 0) {
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
		table_printerror(table, dbret, "c_close");
		return RET_DBERR(dbret);
	}
	return r;
}

retvalue table_removerecord(struct table *table, const char *key, const char *data) {
	int dbret;
	DBT Key, Data;
	DBC *cursor;
	retvalue r;
1160 1161

	SETDBT(Key, key);
1162
	SETDBT(Data, data);