Commit 5189dc09 authored by Raphaël Hertzog's avatar Raphaël Hertzog
Browse files

Generalize the code to fetch Packages/Sources updates 

It's now easy to extend by adding a supplementary lib/foobar-releases.mk
file.

I verified that the set of downloaded files are identical with the
exception of a bug that got fixed in the process: the update-backports-%
rule used to download files for all sections (main, contrib, non-free)
but saved them all under the name corresponding to the main section.
parent 5eb8519d

Makefile

+51 −157
Original line number Diff line number Diff line 
PYTHON = python

PYTHON_MODULES = $(wildcard lib/python/*.py)

BUG_LISTS = $(wildcard data/*/list)



# Adjust these if necessary.  The architecture selection is rather

# arbitrary at the moment.  More architectures can be added later.

# The following variables need to be kept up-to-date and can be adjusted

OLDOLDSTABLE = wheezy

OLDSTABLE    = jessie

STABLE       = stretch

TESTING      = buster



MIRROR = http://debian.csail.mit.edu/debian

SECURITY_MIRROR = http://security.debian.org/debian-security



MIRROR = http://debian.csail.mit.edu/debian/

wheezy_ARCHS = amd64 armel armhf i386

jessie_ARCHS = amd64 arm64 armel armhf i386 mips mipsel powerpc ppc64el s390x

stretch_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x

buster_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x

sid_ARCHS = amd64 arm64 armel armhf hurd-i386 i386 kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x



OLDOLDSTABLE = wheezy

OLDSTABLE    = jessie

STABLE       = stretch

TESTING      = buster

# The rest of the file should not need to be edited



# Include the definitions of the releases to be fetched

include lib/*-releases.mk



# There's a "RELEASES" variable defining the name of all releases to

# process. Then for each release, there are 5 associated variables:

# <name>_MIRROR: the base URL of the mirror hosting the repository

# <name>_DIST: the sub-directory in $MIRROR/dists so that

#              $MIRROR/dists/$DIST/Release is a valid URL

# <name>_ARCHS: the list of architectures supported in this release

# <name>_RELEASE: the release name for the security tracker

# <name>_SUBRELEASE: the sub-release identifier for the security tracker



Q=@

ifneq ($(V),)

Q=

endif



all:

	$(PYTHON) bin/update-db data/security.db
@@ -51,157 +69,33 @@ stamps/DLA-syntax: data/DLA/list bin/check-syntax $(PYTHON_MODULES) 
serve:

	@bash bin/test-web-server



.PHONY: update-packages update-testing update-stable update-oldstable update-oldoldstable

update-packages: update-testing

	set -e ; for rel in sid ; do \

		for archive in main contrib non-free ; do \

		$(PYTHON) bin/apt-update-file \

			$(MIRROR)/dists/$$rel/$$archive/source/Sources \

			data/packages/$${rel}__$${archive}_Sources ; \

		done ; \

	        for arch in $(sid_ARCHS) ; do \

		  for archive in main contrib non-free ; do \

		  $(PYTHON) bin/apt-update-file \

		    $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \

		    data/packages/$${rel}__$${archive}_$${arch}_Packages ; \

		  done ; \

		done ; \

	done



update-testing:

	set -e ; for rel in $(TESTING) ; do \

		for archive in main contrib non-free ; do \

		$(PYTHON) bin/apt-update-file \

			$(MIRROR)/dists/$$rel/$$archive/source/Sources \

			data/packages/$${rel}__$${archive}_Sources ; \

		done ; \

	        for arch in $($(TESTING)_ARCHS) ; do \

		  for archive in main contrib non-free ; do \

		  $(PYTHON) bin/apt-update-file \

		    $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \

		    data/packages/$${rel}__$${archive}_$${arch}_Packages ; \

		  done ; \

		done ; \

	done



update-stable:

	set -e ; for rel in $(STABLE) ; do \

		for archive in main contrib non-free ; do \

		$(PYTHON) bin/apt-update-file \

			$(MIRROR)/dists/$$rel/$$archive/source/Sources \

			data/packages/$${rel}__$${archive}_Sources ; \

		done ; \

	        for arch in $($(STABLE)_ARCHS) ; do \

		  for archive in main contrib non-free ; do \

		  $(PYTHON) bin/apt-update-file \

		    $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \

		    data/packages/$${rel}__$${archive}_$${arch}_Packages ; \

		  done ; \

		done ; \

	done



update-oldstable:

	set -e ; for rel in $(OLDSTABLE) ; do \

		for archive in main contrib non-free ; do \

		$(PYTHON) bin/apt-update-file \

			$(MIRROR)/dists/$$rel/$$archive/source/Sources \

			data/packages/$${rel}__$${archive}_Sources ; \

		done ; \

	        for arch in $($(OLDSTABLE)_ARCHS) ; do \

		  for archive in main contrib non-free ; do \

		  $(PYTHON) bin/apt-update-file \

		    $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \

		    data/packages/$${rel}__$${archive}_$${arch}_Packages ; \

		  done ; \

		done ; \

	done



update-oldoldstable:

	set -e ; for rel in $(OLDOLDSTABLE) ; do \

		for archive in main contrib non-free ; do \

		$(PYTHON) bin/apt-update-file \

			$(MIRROR)/dists/$$rel/$$archive/source/Sources \

			data/packages/$${rel}__$${archive}_Sources ; \

		done ; \

	        for arch in $($(OLDOLDSTABLE)_ARCHS) ; do \

		  for archive in main contrib non-free ; do \

		  $(PYTHON) bin/apt-update-file \

		    $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \

		    data/packages/$${rel}__$${archive}_$${arch}_Packages ; \

		  done ; \

		done ; \

	done



.PHONY: update-security update-old-security update-oldold-security update-testing-security

ST_MIRROR = http://security.debian.org/dists/$(TESTING)/updates

ST_FILE = data/packages/$(TESTING)_security_

update-testing-security:

	for section in main contrib non-free ; do \

	  $(PYTHON) bin/apt-update-file \

	    $(ST_MIRROR)/$$section/source/Sources $(ST_FILE)$${section}_Sources ; \

	  set -e ; for arch in $($(TESTING)_ARCHS) ; do \

	    $(PYTHON) bin/apt-update-file \

	      $(ST_MIRROR)/$$section/binary-$${arch}/Packages $(ST_FILE)$${section}_$${arch}_Packages ; \

	  done ; \

	done



SEC_MIRROR = http://security.debian.org/dists

update-security: update-old-security update-oldold-security

	for archive in $(STABLE); do \

            for section in main contrib non-free ; do \

	    $(PYTHON) bin/apt-update-file \

	      $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \

	      data/packages/$${archive}_security_$${section}_Sources ; \

	    for arch in $($(STABLE)_ARCHS) ; do \

	      $(PYTHON) bin/apt-update-file \

	        $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \

	        data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \

	    done ; \

	  done ; \

	done



update-old-security:

	for archive in $(OLDSTABLE); do \

            for section in main contrib non-free ; do \

	    $(PYTHON) bin/apt-update-file \

	      $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \

	      data/packages/$${archive}_security_$${section}_Sources ; \

	    for arch in $($(OLDSTABLE)_ARCHS) ; do \

	      $(PYTHON) bin/apt-update-file \

	        $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \

	        data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \

	    done ; \

	  done ; \

	done



update-oldold-security:

	for archive in $(OLDOLDSTABLE); do \

.PHONY: update-packages

update-packages: $(foreach release,$(RELEASES),update-$(release))



# This rule is a bit complicated as we need to escape $ for the shell twice,

# once for the eval and once for the usual make processing

define add_update_rule =

.PHONY: update-$(1)

update-$(1):

	$$(Q)set -e; \

	prefix="$$($(1)_RELEASE)_$$($(1)_SUBRELEASE)"; \

	dist="$$($(1)_DIST)"; \

	mirror="$$($(1)_MIRROR)"; \

	for section in main contrib non-free ; do \

	    $(PYTHON) bin/apt-update-file \

	      $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \

	      data/packages/$${archive}_security_$${section}_Sources ; \

	    for arch in $($(OLDOLDSTABLE)_ARCHS) ; do \

	      $(PYTHON) bin/apt-update-file \

	        $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \

	        data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \

	    done ; \

		echo "Fetching $$$$mirror/dists/$$$$dist/$$$$section/source/Sources"; \

		$$(PYTHON) bin/apt-update-file \

		    $$$$mirror/dists/$$$$dist/$$$$section/source/Sources \

		    data/packages/$$$${prefix}_$$$${section}_Sources ; \

	        for arch in $$($(1)_ARCHS) ; do \

			echo "Fetching $$$$mirror/dists/$$$$dist/$$$$section/binary-$$$$arch/Packages"; \

			$$(PYTHON) bin/apt-update-file \

				$$$$mirror/dists/$$$$dist/$$$$section/binary-$$$$arch/Packages \

				data/packages/$$$${prefix}_$$$${section}_$$$${arch}_Packages ; \

		done; \

	done



update-backports: update-backports-$(STABLE) update-backports-$(OLDSTABLE) update-backports-$(OLDOLDSTABLE)



update-backports-%:

	set -e && archive=$(shell echo $@ | cut -d- -f3) ; \

          for section in main contrib non-free ; do \

	    for arch in $($(shell echo $@ | cut -d- -f3)_ARCHS) ; do \

	      $(PYTHON) bin/apt-update-file \

	        $(MIRROR)/dists/$${archive}-backports/$$section/binary-$$arch/Packages \

	        data/packages/$${archive}-backports__main_$${arch}_Packages ; \

	    done ; \

	    $(PYTHON) bin/apt-update-file \

	      $(MIRROR)/dists/$${archive}-backports/$$section/source/Sources \

	      data/packages/$${archive}-backports__main_Sources ; \

	  done ; \

endef

$(foreach release,$(RELEASES),$(eval $(call add_update_rule,$(release))))



update-lists:

	git fetch -q origin && git checkout -f origin/master -- data
@@ -228,4 +122,4 @@ update-compare-nvd: 
	done

	bin/compare-nvd-cve 2> compare-nvd-cve.log



update-all: update-nvd update-lists update-packages update-oldoldstable update-oldstable update-stable update-security update-testing-security update-packages update-backports all
 
update-all: update-nvd update-lists update-packages all

lib/debian-releases.mk

0 → 100644
+39 −0
Original line number Diff line number Diff line 
# This file defines the variables describing all Debian repositories

# that need to be fetched in the "update-packages" process



BACKPORT_RELEASES := $(OLDOLDSTABLE) $(OLDSTABLE) $(STABLE)

SECURITY_RELEASES := $(BACKPORT_RELEASES) $(TESTING)

MAIN_RELEASES := $(SECURITY_RELEASES) sid



# Define the variables for the release on the main mirror

define add_main_release =

$(1)_MIRROR = $$(MIRROR)

$(1)_DIST = $(1)

$(1)_ARCHS ?= amd64 arm64 armel armhf hurd-i386 i386 kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x

$(1)_RELEASE = $(1)

$(1)_SUBRELEASE =

RELEASES += $(1)

endef

$(foreach release,$(MAIN_RELEASES),$(eval $(call add_main_release,$(release))))



# Define the variables for the releases on security.debian.org

define add_security_release =

$(1)_security_MIRROR = $$(SECURITY_MIRROR)

$(1)_security_DIST = $(1)/updates

$(1)_security_ARCHS = $$($(1)_ARCHS)

$(1)_security_RELEASE = $(1)

$(1)_security_SUBRELEASE = security

RELEASES += $(1)_security

endef

$(foreach release,$(SECURITY_RELEASES),$(eval $(call add_security_release,$(release))))



# Define the variables for the *-backports releases

define add_backport_release =

$(1)_backports_MIRROR = $$(MIRROR)

$(1)_backports_DIST = $(1)-backports

$(1)_backports_ARCHS = $$($(1)_ARCHS)

$(1)_backports_RELEASE = $(1)-backports

$(1)_backports_SUBRELEASE =

RELEASES += $(1)_backports

endef

$(foreach release,$(BACKPORT_RELEASES),$(eval $(call add_backport_release,$(release))))