generalize Acquire::GzipIndex (b0f4b486) · Commits · APT Developers / apt

apt-pkg/acquire-item.cc

+29 −15

Original line number	Diff line number	Diff line
		@@ -969,8 +969,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S
		std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
		if (comprExt == "uncompressed")
		Desc.URI = URI;
		else
		else {
		Desc.URI = URI + '.' + comprExt;
		DestFile = DestFile + '.' + comprExt;
		}

		Desc.Description = URIDesc;
		Desc.Owner = this;
		@@ -984,10 +986,11 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S
		/* The only header we use is the last-modified header. */
		string pkgAcqIndex::Custom600Headers()
		{
		std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
		string Final = _config->FindDir("Dir::State::lists");
		Final += URItoFileName(RealURI);
		if (_config->FindB("Acquire::GzipIndexes",false))
		Final += ".gz";
		Final += compExt;

		string msg = "\nIndex-File: true";
		// FIXME: this really should use "IndexTarget::IsOptional()" but that
		@@ -1027,8 +1030,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
		{
		std::string FinalFile = _config->FindDir("Dir::State::lists");
		FinalFile += URItoFileName(URI);
		if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
		FinalFile += ".gz";
		if (_config->FindB("Acquire::GzipIndexes",false) == true)
		FinalFile += '.' + compExt;
		return FinalFile;
		}
		/}}}/
		@@ -1036,8 +1039,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
		void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
		{
		std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
		if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
		DestFile += ".gz";
		if (_config->FindB("Acquire::GzipIndexes",false) == true)
		DestFile += compExt;

		string FinalFile = GetFinalFilename(RealURI, compExt);
		Rename(FinalFile, FileName);
		@@ -1080,7 +1083,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
		/* Always verify the index file for correctness (all indexes must
		* have a Package field) (LP: #346386) (Closes: #627642)
		*/
		FileFd fd(DestFile, FileFd::ReadOnlyGzip);
		FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension);
		// Only test for correctness if the file is not empty (empty is ok)
		if (fd.Size() > 0)
		{
		@@ -1104,8 +1107,8 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
		will work OK */
		DestFile = _config->FindDir("Dir::State::lists") + "partial/";
		DestFile += URItoFileName(RealURI);
		if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
		DestFile += ".gz";
		if (_config->FindB("Acquire::GzipIndexes",false))
		DestFile += '.' + compExt;

		// Remove the compressed version.
		if (Erase == true)
		@@ -1146,16 +1149,23 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
		// matching the Release file
		if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
		{
		// set destfile to the final destfile
		if(_config->FindB("Acquire::GzipIndexes",false) == false)
		{
		DestFile = _config->FindDir("Dir::State::lists") + "partial/";
		DestFile += URItoFileName(RealURI);
		}

		ReverifyAfterIMS(FileName);
		return;
		}
		string decompProg;

		// If we enable compressed indexes, queue for hash verification
		if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local)
		if (_config->FindB("Acquire::GzipIndexes",false))
		{
		DestFile = _config->FindDir("Dir::State::lists");
		DestFile += URItoFileName(RealURI) + ".gz";
		DestFile += URItoFileName(RealURI) + '.' + compExt;

		Decompression = true;
		Desc.URI = "copy:" + FileName;
		@@ -1201,11 +1211,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire Owner, IndexTarget const Target,
		// ---------------------------------------------------------------------
		string pkgAcqIndexTrans::Custom600Headers()
		{
		std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
		string Final = _config->FindDir("Dir::State::lists");
		Final += URItoFileName(RealURI);

		if (_config->FindB("Acquire::GzipIndexes",false))
		Final += ".gz";
		Final += compExt;

		struct stat Buf;
		if (stat(Final.c_str(),&Buf) != 0)
		@@ -1552,10 +1562,14 @@ void pkgAcqMetaIndex::AuthDone(string Message) /{{{/
		URItoFileName((*Target)->URI);
		unlink(index.c_str());
		// and also old gzipindexes
		index += ".gz";
		std::vector<std::string> types = APT::Configuration::getCompressionTypes();
		for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
		{
		index += '.' + (*t);
		unlink(index.c_str());
		}
		}
		}


		// Download further indexes with verification

apt-pkg/deb/debindexfile.cc

+38 −22

Original line number	Diff line number	Diff line
		@@ -80,14 +80,18 @@ pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const
		{
		string SourcesURI = _config->FindDir("Dir::State::lists") +
		URItoFileName(IndexURI("Sources"));
		string SourcesURIgzip = SourcesURI + ".gz";

		if (!FileExists(SourcesURI) && !FileExists(SourcesURIgzip))
		return NULL;
		else if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip))
		SourcesURI = SourcesURIgzip;

		std::vector<std::string> types = APT::Configuration::getCompressionTypes();
		for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
		{
		string p;
		p = SourcesURI + '.' + *t;
		if (FileExists(p))
		return new debSrcRecordParser(p, this);
		}
		if (FileExists(SourcesURI))
		return new debSrcRecordParser(SourcesURI, this);
		return NULL;
		}
		/}}}/
		// SourcesIndex::Describe - Give a descriptive path to the index /{{{/
		@@ -129,10 +133,14 @@ string debSourcesIndex::Info(const char *Type) const
		inline string debSourcesIndex::IndexFile(const char *Type) const
		{
		string s = URItoFileName(IndexURI(Type));
		string sgzip = s + ".gz";
		if (!FileExists(s) && FileExists(sgzip))
		return sgzip;
		else

		std::vector<std::string> types = APT::Configuration::getCompressionTypes();
		for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
		{
		string p = s + '.' + *t;
		if (FileExists(p))
		return p;
		}
		return s;
		}

		@@ -259,10 +267,14 @@ string debPackagesIndex::Info(const char *Type) const
		inline string debPackagesIndex::IndexFile(const char *Type) const
		{
		string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
		string sgzip = s + ".gz";
		if (!FileExists(s) && FileExists(sgzip))
		return sgzip;
		else

		std::vector<std::string> types = APT::Configuration::getCompressionTypes();
		for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
		{
		string p = s + '.' + *t;
		if (FileExists(p))
		return p;
		}
		return s;
		}
		string debPackagesIndex::IndexURI(const char *Type) const
		@@ -411,10 +423,14 @@ debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section
		inline string debTranslationsIndex::IndexFile(const char *Type) const
		{
		string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
		string sgzip = s + ".gz";
		if (!FileExists(s) && FileExists(sgzip))
		return sgzip;
		else

		std::vector<std::string> types = APT::Configuration::getCompressionTypes();
		for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
		{
		string p = s + '.' + *t;
		if (FileExists(p))
		return p;
		}
		return s;
		}
		string debTranslationsIndex::IndexURI(const char *Type) const

apt-private/private-update.cc

+8 −1

Original line number	Diff line number	Diff line
		@@ -56,10 +56,17 @@ bool DoUpdate(CommandLine &CmdL)
		if (List->GetIndexes(&Fetcher,true) == false)
		return false;

		std::string compExt = APT::Configuration::getCompressionTypes()[0];
		pkgAcquire::UriIterator I = Fetcher.UriBegin();
		for (; I != Fetcher.UriEnd(); ++I)
		c1out << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' <<
		{
		std::string FileName = flNotDir(I->Owner->DestFile);
		if(compExt.empty() == false &&
		APT::String::Endswith(FileName, compExt))
		FileName = FileName.substr(0, FileName.size() - compExt.size() - 1);
		c1out << '\'' << I->URI << "' " << FileName << ' ' <<
		I->Owner->FileSize << ' ' << I->Owner->HashSum() << std::endl;
		}
		return true;
		}

methods/copy.cc

+4 −7

Original line number	Diff line number	Diff line
		@@ -37,15 +37,12 @@ class CopyMethod : public pkgAcqMethod

		void CopyMethod::CalculateHashes(FetchResult &Res)
		{
		// For gzip indexes we need to look inside the gzip for the hash
		// We can not use the extension here as its not used in partial
		// on a IMS hit
		FileFd::OpenMode OpenMode = FileFd::ReadOnly;
		Hashes Hash;
		FileFd::CompressMode CompressMode = FileFd::None;
		if (_config->FindB("Acquire::GzipIndexes", false) == true)
		OpenMode = FileFd::ReadOnlyGzip;
		CompressMode = FileFd::Extension;

		Hashes Hash;
		FileFd Fd(Res.Filename, OpenMode);
		FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode);
		Hash.AddFD(Fd);
		Res.TakeHashes(Hash);
		}

test/integration/test-bug-595691-empty-and-broken-archive-files

+1 −1

Original line number	Diff line number	Diff line
		@@ -121,7 +121,7 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http"
		testaptgetupdate "Get: http://localhost:8080 Packages
		Err http://localhost:8080 Packages
		Empty files can't be valid archives
		W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives
		W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages.${COMPRESS}) Empty files can't be valid archives

		E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http"
		}