debian/changelog

@@ -5,6 +5,8 @@ blends (0.6.103) UNRELEASED; urgency=medium
      - blends-dev: Depends: ${python3:Depends}
   * d/rules: --with python3
   * ignore false lintian warning binary-package-depends-on-toolchain-package
+  * cme fix dpkg-control
+  * Take over tasks_diff from blends-gsoc to create dependency_data
 
  -- Andreas Tille <tille@debian.org>  Wed, 28 Mar 2018 16:22:10 +0200
 

debian/control

@@ -4,7 +4,7 @@ Uploaders: Petter Reinholdtsen <pere@debian.org>,
            Andreas Tille <tille@debian.org>,
            Jonas Smedegaard <dr@jones.dk>,
            Ole Streicher <olebole@debian.org>,
-           Mike Gabriel <sunweaver@debian.org>,
+           Mike Gabriel <sunweaver@debian.org>
 Section: devel
 Priority: optional
 Build-Depends: debhelper (>= 10)
@@ -13,7 +13,7 @@ Build-Depends-Indep: xmlto,
                      w3m,
                      dh-python,
                      python3-all
-Standards-Version: 4.0.0
+Standards-Version: 4.1.3
 Vcs-Browser: https://salsa.debian.org/blends-team/blends
 Vcs-Git: https://salsa.debian.org/blends-team/blends.git
 
@@ -25,7 +25,6 @@ Depends: debconf,
          ${misc:Depends},
          ${python3:Depends}
 Suggests: blends-doc
-Replaces: cdd-dev
 Description: Debian Pure Blends common files for developing metapackages
  This package makes life easier when packaging metapackages.  Perhaps
  this will also encourage other people to build metapackages if there are
@@ -54,7 +53,6 @@ Section: doc
 Depends: ${misc:Depends}
 Suggests: www-browser,
           postscript-viewer
-Replaces: cdd-doc
 Description: Debian Pure Blends documentation
  This paper is intended to people who are interested in the philosophy
  of Debian Pure Blends and the technique which is used to
@@ -69,8 +67,8 @@ Description: Debian Pure Blends documentation
 
 Package: blends-tasks
 Architecture: all
-Priority: important
 Section: misc
+Priority: important
 Depends: ${misc:Depends},
          tasksel
 Description: Debian Pure Blends tasks for new installations

devtools/Makefile

@@ -10,13 +10,34 @@
 # sources.list file available
 TARGET_DIST := $(shell head -1 debian/changelog |awk '{print $$3}'|tr -d ';')
 BLEND := $(shell /usr/share/blends-dev/blend-get-names blendname)
+VERSION  := $(shell dpkg-parsechangelog -ldebian/changelog | grep Version: | cut -f2 -d' ' | cut -f1 -d- )
 GENCONTROL := /usr/share/blends-dev/blend-gen-control
+TASKSDIFF := /usr/share/blends-dev/tasks_diff
+DEPENDENCIES_DATA := dependency_data
 GENCONTROL_OPTS := 
 TASKSELOPTS := $(shell grep TASKSELOPTS Makefile | cut -d '=' -f2)
 
 # Verify whether config/control exists, if yes, add it to the depends of debian/control
 CONFIGCONTROL := $(shell if [ -d config -a -e config/control ] ; then echo config/control; fi)
 
+#get two latest releases
+RELEASES   := $(shell grep '^$(BLEND)' debian/changelog | head -2 | awk '{print $$2}' | tr -d '[(,)]')
+LATEST    := $(shell echo "$(RELEASES)" | cut -d ' ' -f1)
+PREVIOUS   := $(shell echo $(RELEASES) | cut -d ' ' -f2)
+
+ifneq "$(LATEST)" "$(PREVIOUS)"
+	LINEEND   := $(shell lineend=`grep '^$(BLEND)' debian/changelog -n | sed -n 2p | cut -d ':' -f1`; echo "$$lineend - 3" | bc)
+	LINESTART  := $(shell linestart=`grep "* start of automatic changelog entry *" debian/changelog -n | head -1 | awk '{print $$1}' | tr -d ':'`; if [ -z "$$linestart" ]; then echo "0"; else echo "$$linestart - 1" | bc; fi)
+
+	ISGREATER := $(shell expr $(LINESTART) \> $(LINEEND))
+
+	ifeq "$(LINESTART)" "0"
+		LINESTART := $(LINEEND)
+	else ifeq "$(ISGREATER)" "1"
+		LINESTART := $(LINEEND)
+	endif
+endif
+
 all: $(BLEND)-tasks.desc debian/control
 
 debian/control: debian/control.stub debian/changelog tasks/* $(CONFIGCONTROL)
@@ -26,6 +47,34 @@ tasksel: $(BLEND)-tasks.desc
 $(BLEND)-tasks.desc: tasks/* debian/changelog
 	$(GENCONTROL) $(TASKSELOPTS) -r $(TARGET_DIST) -S -t
 
+dependency_data:
+	if [ ! -d $(DEPENDENCIES_DATA) ]; then \
+		echo "$(DEPENDENCIES_DATA) directory does not exist, creating it now."; \
+		mkdir $(DEPENDENCIES_DATA); \
+	fi
+
+statusdump: dependency_data
+	$(TASKSDIFF) --status-dump --tasks .  --output $(DEPENDENCIES_DATA)/$(BLEND)_$(VERSION).json
+
+#update changelog with dependencies changes
+changelogentry: debian/changelog statusdump
+ifneq "$(LATEST)" "$(PREVIOUS)"
+	if [ ! -f $(DEPENDENCIES_DATA)/$(BLEND)_$(LATEST).json ]; then \
+		echo "$(DEPENDENCIES_DATA)/$(BLEND)_$(LATEST).json does not exist, can not generate changelog dependencies-changes entry"; \
+		exit -1; \
+	fi
+	if [ ! -f $(DEPENDENCIES_DATA)/$(BLEND)_$(PREVIOUS).json ]; then \
+		echo "$(DEPENDENCIES_DATA)/$(BLEND)_$(PREVIOUS).json does not exist, can not generate changelog dependencies-changes entry"; \
+		exit -1; \
+	fi
+
+	(sed $(LINESTART)q debian/changelog; \
+	 $(TASKSDIFF) --compare $(CURRENTPARH)/$(DEPENDENCIES_DATA)/$(BLEND)_$(LATEST).json,$(CURRENTPARH)/$(DEPENDENCIES_DATA)/$(BLEND)_$(PREVIOUS).json | sed 's/^/  /'; \
+	 sed -n '$(LINEEND),$$p' debian/changelog; ) > debian/changelog.new && mv debian/changelog.new debian/changelog
+else
+	echo "It is the first release, skip the changelog entry"
+endif
+
 packages.txt: tasks/*
 	$(GENCONTROL) -r $(TARGET_DIST) -a > packages.txt.$$$$ && mv packages.txt.$$$$ packages.txt
 

devtools/tasks_diff

+#!/usr/bin/python3
+
+# Copyright 2013: Emmanouil Kiagias <e.kiagias@gmail.com>
+# Converted via 2to3 by Andreas Tille <tille@debian.org>
+# License: GPL
+
+"""
+no  documentation for the moment
+"""
+
+import os
+import re
+import sys
+import json
+import pprint
+import logging
+import argparse
+import subprocess
+from debian import deb822
+
+#with this we distinguish the start of automatic entry in the changelog so we
+#can replace the entry if needed
+START_FLAG = "* start of automatic changelog entry *"
+
+def clean_up_packages(packages):
+	logger = logging.getLogger(__name__)
+	# Hack: Debian Edu tasks files are using '\' at EOL which is broken
+	#       in RFC 822 files, but blend-gen-control from blends-dev relies
+	#       on this.  So remove this stuff here for the Moment
+	pkgs = re.sub('\\\\\n\s+', '', packages)
+
+	# Remove versions from versioned depends
+	pkgs = re.sub(' *\([ ><=\.0-9]+\) *', '', pkgs)
+
+	# temporary strip spaces from alternatives ('|') to enable erroneous space handling as it was done before
+	pkgs = re.sub('\s*\|\s*', '|', pkgs)
+
+	# turn alternatives ('|') into real depends for this purpose
+	# because we are finally interested in all alternatives
+	pkgslist = pkgs.split(',')
+	# Collect all dependencies in one line first,
+	# create an object for each later
+	pkgs_in_one_line = []
+	for depl in pkgslist:
+		dl = depl.strip()
+		if dl != '': # avoid confusion when ',' is at end of line
+			if re.search('\s', dl):
+				#logger.error("Blend %s task %s: Syntax error '%s'" % (blend, task, dl))
+				# trying to fix the syntax error after issuing error message
+				dlspaces = re.sub('\s+', ',', dl).split(',')
+				for dls in dlspaces:
+					pkgs_in_one_line.append(dls.strip())
+					#logger.info("Blend %s task %s: Found '%s' package inside broken syntax string - please fix task file anyway" % (blend, task, dls.strip()))
+			else:
+				# in case we have to deal with a set of alternatives
+				if re.search('\|', dl):
+					#for da in dl.split('|'):
+					#  deps_in_one_line.append(da)
+					dl = re.sub('\|', ' | ', dl)
+				pkgs_in_one_line.append(dl)
+				# self.inject_package_alternatives(blend, task, strength, dl)
+
+	return pkgs_in_one_line
+
+def load_task(path_to_task):
+	"""
+	parses a task file and return a dictionary containing all its package headers elements
+	(depends, suggests etc)
+	"""
+	ftask = open(path_to_task, 'r')
+	task = os.path.basename(path_to_task)
+	taskinfo = {}
+
+	for header in ["depends", "suggests", "recommends", "ignore", "avoid"]:
+		taskinfo[header] = []
+
+	for paragraph in deb822.Sources.iter_paragraphs(ftask, shared_storage=False):
+		if "depends" in paragraph:
+			taskinfo["depends"] += clean_up_packages(paragraph["depends"])
+
+		if "suggests" in paragraph:
+			taskinfo["suggests"] += clean_up_packages(paragraph["suggests"])
+
+		if "recommends" in paragraph:
+			taskinfo["recommends"] += clean_up_packages(paragraph["recommends"])
+
+		if "ignore" in paragraph:
+			taskinfo["ignore"] += clean_up_packages(paragraph["ignore"])
+
+		if "avoid" in paragraph:
+			taskinfo["avoid"] += clean_up_packages(paragraph["avoid"])
+
+	return task, taskinfo
+
+def compare_tasks(tasks, tasks_compare, taskprefix):
+	"""
+	This function will dump in stdout the package differences between
+	the given tasks1 and tasks2
+	"""
+
+	first_print = True
+
+	for task in tasks:
+		if not task in tasks_compare:
+			continue
+		
+		task_first = True 
+		first_add = True
+		for header in ["depends", "recommends", "suggests", "ignore", "avoid"]:
+			added =  set(tasks[task][header]) - set(tasks_compare[task][header])
+			if added:
+				if first_print:
+					print(START_FLAG, "\n")
+					print("* Changes in metapackage dependencies")
+					first_print = False
+				if task_first:
+					print(" -{0}-{1}".format(taskprefix,task))
+					task_first = False
+				if first_add:
+					print("  added:")
+					first_add = False
+				print("    {0}: ".format(header.capitalize()), end=' ')
+				print(", ".join(added))
+		
+		first_remove = True
+		for header in ["depends", "recommends", "suggests", "ignore", "avoid"]:
+			removed =  set(tasks_compare[task][header]) - set(tasks[task][header])
+			if removed:
+				if first_print:
+					print(START_FLAG, "\n")
+					print("* Changes in metapackage dependencies")
+					first_print = False
+				if task_first:
+					print(" -{0}-{1}".format(taskprefix,task))
+					task_first = False
+				if first_remove:
+					print("  removed:")
+					first_remove = False
+				print("    {0}: ".format(header.capitalize()), end=' ')
+				print(", ".join(removed))
+
+	removed_tasks =  set(tasks_compare.keys()) - set(tasks.keys())
+	added_tasks =  set(tasks.keys()) - set(tasks_compare.keys())
+	if added_tasks:
+		if first_print:
+			print(START_FLAG, "\n")
+			print("* Changes in metapackage dependencies")
+			first_print = False
+		print("* New metapackages:")
+		for newtask in added_tasks:
+			print(" -{0}-{1}".format(taskprefix, newtask))
+
+	if removed_tasks:
+		if first_print:
+			print(START_FLAG, "\n")
+			print("* Changes in metapackage dependencies")
+			first_print = False
+		print("* Removed metapackages:")
+		for removedtask in removed_tasks:
+			print(" -{0}-{1}".format(taskprefix, removedtask))
+
+def load_tasks(tasks_path):
+	tasks = {}
+
+	for taskpath in tasks_path:
+		taskname, taskinfo = load_task(taskpath)
+		tasks[taskname] = taskinfo
+
+	return tasks
+
+if __name__ == "__main__":
+	blend_dev_dir = "/usr/share/blends-dev/"
+	default_json = "tasks.json"
+
+	##TODO add proper epilog giving example usage
+	parser = argparse.ArgumentParser(epilog="")
+	
+	parser.add_argument("-t", "--tasks", dest="tasks", type=str,
+	                    help="Path to task files", default=".")
+	parser.add_argument("-s", "--status-dump", dest="statusdump", action="store_true",
+						help="Dump dependencies status into a json file")
+	parser.add_argument("-o", "--output", dest="output", type=str, default=default_json,
+						help="Output file where to store the dependencies json file(when -s/--status-dump is provided)")
+	parser.add_argument("-c", "--compare", dest="compare", type=str,
+						help="Provide two comma separated(without spaces)  paths to json files to be compared")
+	parser.add_argument("-d", "--debug", dest="debug", action="store_true", default=False,
+	                    help="Print debug information")
+	#parse the command line arguments
+	args = parser.parse_args()
+
+	if args.debug:
+	    logging.basicConfig(level=logging.DEBUG)
+	else:
+	    logging.basicConfig()
+	logger = logging.getLogger(__name__)
+
+	#load the taskprefix
+	taskprefix = ""
+	if not args.statusdump and args.compare:
+		command = blend_dev_dir+"blend-get-names metapackageprefix"
+		process = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
+		output = process.communicate()[0].strip()
+		if process.returncode != 0:
+			logger.error(output)
+			sys.exit(-1)
+		else:
+			taskprefix = output
+
+	if not args.statusdump and not args.compare:
+		logger.error("At least -s/--statusdump or -c/--compare argument must be provided")
+		sys.exit(-1)
+
+	path_to_tasks = os.path.join(args.tasks, "tasks")
+	if not os.path.isdir(path_to_tasks):
+		logger.error("tasks directory could not be found in given path. aborting...")
+		sys.exit(-1)
+
+	logger.debug("Reading task files from directory {0}".format(path_to_tasks))
+	tasks = [ os.path.join(path_to_tasks, fold) for fold in os.listdir(path_to_tasks) if not fold.startswith('.') ]
+	giventasks = load_tasks(tasks)
+
+	if args.statusdump:
+		logger.debug("Status dump was selected")
+
+		with open(args.output, "w") as fout:
+			logger.debug("Dumping json dependencies file into {0}".format(args.output))
+			json.dump(giventasks, fout)
+
+		sys.exit(0)
+
+	if args.compare:
+		if not ',' in args.compare:
+			logger.error("For --compare two comma separated paths to json files should be provided.")
+			sys.exit(-1)
+
+		latest, previous = [ x.strip() for x in args.compare.split(',') ]
+
+		if not os.path.isfile(previous) or not os.path.isfile(latest):
+			logger.error("Please provide existing json files to be compared.")
+			sys.exit(-1)
+
+		logger.debug("Comparing json files:")
+		logger.debug("{0} with {1}".format(latest, previous))
+
+		latest_tasks = json.load(open(latest))
+		previous_tasks = json.load(open(previous))
+
+		logger.debug("Comparing releases...")
+		compare_tasks(latest_tasks, previous_tasks, taskprefix)
+