debian/README.source

+firmware-free
+=============
+
+Upstream
+--------
+
+firmware-free is based on the linux-firmware.git repository, which
+does not currently make tarball (or tagged) releases.  Also, most of
+its contents are non-free.
+git://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git
+
+We set the upstream version to the date of the last commit and use
+uscan to generate a tarball from upstream git.  The Files-Excluded
+field in 'debian/copyright' lists file patterns that should be
+excluded from the tarball.
+
+We also pull module information from Linux binary packages.  To update
+this information, run:
+
+    debian/bin/update-modinfo
+
+You can optionally pass a specific kernel release string or directory
+in which to look for modules.
+
+Licences
+--------
+
+The upstream source includes the file 'WHENCE' which lists the licence
+and any source code for each file.  The script
+'debian/bin/check_upstream.py' will warn about any files that aren't
+recognised to be free based on the information in 'WHENCE' and that
+haven't been excluded.
+
+Binary package definition
+-------------------------
+
+The binary package definitions are placed under 'debian/config' and
+written in a format similar to INI files, parsed using the Python
+ConfigParser module
+<https://docs.python.org/3/library/configparser.html>,
+
+The [base] packages field in the top-level 'defines' file lists all
+the binary package names, without the fixed prefix 'firmware-'.
+
+For each binary package, there is a subdirectory with matching name
+(again without the prefix) containing another 'defines' file with
+metadata.  Any non-upstream firmware files are also included in
+these subdirectories.
+
+The script 'debian/bin/check_upstream.py' will list the upstream
+firmware files that are free but not assigned to any binary
+package.
+
+Required metadata:
+
+[base] desc: Short description of related hardware, used in Description
+[base] files: List of files to be included, one per line
+[base] longdesc: Longer description of related hardware and drivers, used in
+                 Description.
+
+Optional metadata:
+
+[base] uri: Upstream URL, used as Homepage and in copyright file.  Not
+            needed if the contents come from linux-firmware.git.
+[base] support: Set to 'initramfs-tools' if update-initramfs should be
+                invoked after installation
+[base] replaces: Used as Replaces field
+[base] conflicts: Used as Conflicts field
+[base] provides: Used as Provides field
+
+Optional per-file metadata:
+
+[<filename>_base] desc: One-line description for this file, used in
+                        package description
+[<filename>_base] version: Verson number for this file, used in package
+                           description
+
+To re-generate debian/control (and other files) based on these
+definitions:
+
+1. Install the current linux-support-<kernelversion> package
+2. Ensure debian/rules.defs has the right value of KERNELVERSION
+3. If the 'orig.tar' archive is not yet available, retrieve it as follows:
+   ``uscan --download-version <upstream-version>`` (<upstream-version> is f.e. ``20230210``)
+   To retrieve <upstream-version> programmatically, use f.e. this:
+   $(head -n1 debian/changelog | sed 's/.*\([[:digit:]]\{8\}\).*/\1/')
+   Combined that results in:
+   ``uscan --download-version $(head -n1 debian/changelog | sed 's/.*\([[:digit:]]\{8\}\).*/\1/')``
+4. Run: debian/rules orig
+5. Run: debian/rules debian/control
+
+To update the current package's changelog with the new upstream version:
+
+1. Run: PYTHONPATH=/usr/share/linux-support-<version>/lib/python \
+            debian/bin/release-update /path/to/linux-firmware-repository
+
+   This will update debian/changelog with the changes added between
+   the commit of the current package version and the HEAD commit found in the
+   linux firmware repository, passed as argument.
+
+2. Remove entries regarding firmwares files not packaged
+
+ -- Ben Hutchings <benh@debian.org>, Sun, 20 Aug 2023 18:33:40 +0200

debian/bin/gencontrol.py

 #!/usr/bin/env python3
 
-import os, re, sys, locale
+import io
+import json
+import locale
+import os
+import re
+import sys
 
 sys.path.insert(0, "debian/lib/python")
 sys.path.append(sys.argv[1] + "/lib/python")
 locale.setlocale(locale.LC_CTYPE, "C.UTF-8")
 
 from config import Config
-from debian_linux.debian import Package, PackageRelation
+from debian_linux.debian import BinaryPackage, PackageRelation, _ControlFileDict
 from debian_linux.debian import PackageDescription as PackageDescriptionBase
 import debian_linux.gencontrol
 from debian_linux.gencontrol import Makefile, MakeFlags, PackagesList
 from debian_linux.utils import TextWrapper
-from debian_linux.utils import Templates as TemplatesBase, read_control
+from debian_linux.utils import Templates as TemplatesBase
 from collections import OrderedDict
 
 class PackageDescription(PackageDescriptionBase):
@@ -49,92 +54,20 @@ class PackageDescription(PackageDescriptionBase):
             for i in desc:
                 self.append(i)
 
-Package._fields['Description'] = PackageDescription
+BinaryPackage._fields['Description'] = PackageDescription
 
-class Template(dict):
+class Template(_ControlFileDict):
     _fields = OrderedDict((
         ('Template', str),
         ('Type', str),
         ('Default', str),
-        ('Description', PackageDescription),
+        ('Description', PackageDescriptionBase),
     ))
 
-    def __setitem__(self, key, value):
-        try:
-            cls = self._fields[key]
-            if not isinstance(value, cls):
-                value = cls(value)
-        except KeyError: pass
-        super(Template, self).__setitem__(key, value)
-
-    def keys(self):
-        keys = set(super(Template, self).keys())
-        for i in self._fields.keys():
-            if i in self:
-                keys.remove(i)
-                yield i
-        for i in keys:
-            yield i
-
-    def items(self):
-        for i in self.keys():
-            yield (i, self[i])
-
-    def values(self):
-        for i in self.keys():
-            yield self[i]
-
 
 class Templates(TemplatesBase):
-    # TODO
-    def _read(self, name):
-        prefix, id = name.split('.', 1)
-
-        for dir in self.dirs:
-            filename = "%s/%s.in" % (dir, name)
-            if os.path.exists(filename):
-                f = open(filename, 'r')
-                mode = os.stat(f.fileno()).st_mode
-                if prefix == 'control':
-                    return (read_control(f), mode)
-                elif prefix == 'templates':
-                    return (self._read_templates(f), mode)
-                return (f.read(), mode)
-
-    def _read_templates(self, f):
-        entries = []
-
-        while True:
-            e = Template()
-            last = None
-            lines = []
-            while True:
-                line = f.readline()
-                if not line:
-                    break
-                line = line.strip('\n')
-                if not line:
-                    break
-                if line[0] in ' \t':
-                    if not last:
-                        raise ValueError('Continuation line seen before first header')
-                    lines.append(line.lstrip())
-                    continue
-                if last:
-                    e[last] = '\n'.join(lines)
-                i = line.find(':')
-                if i < 0:
-                    raise ValueError("Not a header, not a continuation: ``%s''" % line)
-                last = line[:i]
-                lines = [line[i+1:].lstrip()]
-            if last:
-                e[last] = '\n'.join(lines)
-            if not e:
-                break
-
-            entries.append(e)
-
-        return entries
+    def get_templates_control(self, key: str, context: dict[str, str] = {}) -> Template:
+        return Template.read_rfc822(io.StringIO(self.get(key, context)))
 
 
 class GenControl(debian_linux.gencontrol.Gencontrol):
@@ -142,18 +75,44 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
         self.config = Config()
         self.templates = Templates()
 
+        with open('debian/modinfo.json', 'r') as f:
+            self.modinfo = json.load(f)
+
+        # Make another dict keyed by firmware names
+        self.firmware_modules = {}
+        for name, info  in self.modinfo.items():
+            for firmware_filename in info['firmware']:
+                self.firmware_modules.setdefault(firmware_filename, []) \
+                                     .append(name)
+
     def __call__(self):
         packages = PackagesList()
         makefile = Makefile()
 
         self.do_source(packages)
+        self.do_extra(packages, makefile)
         self.do_main(packages, makefile)
 
         self.write(packages, makefile)
 
     def do_source(self, packages):
-        source = self.templates["control.source"]
-        packages['source'] = self.process_package(source[0], ())
+        packages['source'] = self.templates.get_source_control("control.source", {})[0]
+
+    def do_extra(self, packages, makefile):
+        config_entry = self.config['base',]
+        vars = {}
+        vars.update(config_entry)
+
+        for package_binary in self.templates.get_control("control.extra", {}):
+            assert package_binary['Package'].startswith('firmware-')
+            package = package_binary['Package'].replace('firmware-', '')
+
+            makeflags = MakeFlags()
+            makeflags['FILES'] = ''
+            makeflags['PACKAGE'] = package
+            makefile.add_cmds('binary-indep', ["$(MAKE) -f debian/rules.real binary-indep %s" % makeflags])
+
+            packages.append(package_binary)
 
     def do_main(self, packages, makefile):
         config_entry = self.config['base',]
@@ -162,6 +121,9 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
 
         makeflags = MakeFlags()
 
+        for i in ('build', 'binary-arch', 'setup'):
+            makefile.add_cmds("%s_%%" % i, ["@true"])
+
         for package in config_entry['packages']:
             self.do_package(packages, makefile, package, vars.copy(), makeflags.copy())
 
@@ -169,10 +131,15 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
         config_entry = self.config['base', package]
         vars.update(config_entry)
         vars['package'] = package
+        vars['package-env-prefix'] = 'FIRMWARE_' + package.upper().replace('-', '_')
 
         makeflags['PACKAGE'] = package
 
-        binary = self.templates["control.binary"]
+        # Those might be absent, set them to empty string for replacement to work:
+        empty_list = ['replaces', 'conflicts', 'breaks', 'provides', 'recommends']
+        for optional in ['replaces', 'conflicts', 'breaks', 'provides', 'recommends']:
+            if optional not in vars:
+                vars[optional] = ''
 
         package_dir = "debian/config/%s" % package
 
@@ -184,6 +151,7 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
 
         files_orig = config_entry['files']
         files_real = {}
+        files_unused = []
         links = {}
         links_rev = {}
 
@@ -204,13 +172,20 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
                 f1 = f.rsplit('-', 1)
                 if f in files_orig:
                     files_real[f] = f, cur_path, None
-                elif len(f1) > 1:
+                    continue
+                if len(f1) > 1:
                     f_base, f_version = f1
                     if f_base in files_orig:
                         if f_base in files_real:
                             raise RuntimeError("Multiple files for %s" % f_base)
                         files_real[f_base] = f_base, package_dir + '/' + f, \
                                              f_version
+                        continue
+                # Whitelist files not expected to be installed as firmware
+                if f in ['defines', 'LICENSE.install',
+                         'update.py', 'update.sh']:
+                    continue
+                files_unused.append(f)
 
         # Take all the other files from upstream
         for f in files_orig:
@@ -224,15 +199,20 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
             link_target = os.path.normpath(os.path.join(f, '..', links[f]))
             links_rev.setdefault(link_target, []).append(f)
 
-        makeflags['FILES'] = ' '.join(["%s:%s" % (i[1], i[0]) for i in files_real.values()])
+        if files_unused:
+            print('W: %s: unused files:' % package, ' '.join(files_unused),
+                  file=sys.stderr)
+
+        makeflags['FILES'] = ' '.join(["%s:%s" % (i[1], i[0]) for i in sorted(files_real.values())])
         vars['files_real'] = ' '.join(["/lib/firmware/%s" % i for i in config_entry['files']])
 
         makeflags['LINKS'] = ' '.join(["%s:%s" % (link, target)
-                                       for link, target in links.items()])
+                                       for link, target in sorted(links.items())])
 
         files_desc = ["Contents:"]
-        firmware_meta_temp = self.templates["metainfo.xml.firmware"]
+        firmware_meta_temp = self.templates.get("metainfo.xml.firmware")
         firmware_meta_list = []
+        module_names = set()
 
         wrap = TextWrapper(width = 71, fix_sentence_endings = True,
                            initial_indent = ' * ',
@@ -240,6 +220,8 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
         for f in config_entry['files']:
             firmware_meta_list.append(self.substitute(firmware_meta_temp,
                                                       {'filename': f}))
+            for module_name in self.firmware_modules.get(f, []):
+                module_names.add(module_name)
             if f in links:
                 continue
             f, f_real, version = files_real[f]
@@ -259,23 +241,32 @@ class GenControl(debian_linux.gencontrol.Gencontrol):
                 desc = "%s" % f
             files_desc.extend(wrap(desc))
 
-        packages_binary = self.process_packages(binary, vars)
+        modaliases = set()
+        for module_name in module_names:
+            for modalias in self.modinfo[module_name]['alias']:
+                modaliases.add(modalias)
+        modalias_meta_list = [
+            self.substitute(self.templates.get("metainfo.xml.modalias"),
+                            {'alias': alias})
+            for alias in sorted(list(modaliases))
+        ]
+
+        packages_binary = self.templates.get_control("control.binary", vars)
 
         packages_binary[0]['Description'].append_pre(files_desc)
 
         if 'initramfs-tools' in config_entry.get('support', []):
-            postinst = self.templates['postinst.initramfs-tools']
+            postinst = self.templates.get('postinst.initramfs-tools')
             open("debian/firmware-%s.postinst" % package, 'w').write(self.substitute(postinst, vars))
 
         if 'license-accept' in config_entry:
             license = open("%s/LICENSE.install" % package_dir, 'r').read()
-            preinst = self.templates['preinst.license']
+            preinst = self.templates.get('preinst.license')
             preinst_filename = "debian/firmware-%s.preinst" % package
             open(preinst_filename, 'w').write(self.substitute(preinst, vars))
 
-            templates = self.process_templates(self.templates['templates.license'], vars)
-            license_split = re.split(r'\n\s*\n', license)
-            templates[0]['Description'].extend(license_split)
+            templates = self.templates.get_templates_control('templates.license', vars)
+            templates[0]['Description'].append(re.sub('\n\n', '\n.\n', license))
             templates_filename = "debian/firmware-%s.templates" % package
             self.write_rfc822(open(templates_filename, 'w'), templates)
 
@@ -288,14 +279,15 @@ You must agree to the terms of this license before it is installed."""
 
         packages.extend(packages_binary)
 
-        makefile.add('binary-indep', cmds = ["$(MAKE) -f debian/rules.real binary-indep %s" % makeflags])
+        makefile.add_cmds('binary-indep', ["$(MAKE) -f debian/rules.real binary-indep %s" % makeflags])
 
         vars['firmware-list'] = ''.join(firmware_meta_list)
+        vars['modalias-list'] = ''.join(modalias_meta_list)
         # Underscores are preferred to hyphens
         vars['package-metainfo'] = package.replace('-', '_')
         # Summary must not contain line breaks
         vars['longdesc-metainfo'] = re.sub(r'\s+', ' ', vars['longdesc'])
-        package_meta_temp = self.templates["metainfo.xml"]
+        package_meta_temp = self.templates.get("metainfo.xml", {})
         # XXX Might need to escape some characters
         open("debian/firmware-%s.metainfo.xml" % package, 'w').write(self.substitute(package_meta_temp, vars))
 

debian/bin/update-modinfo

+#!/usr/bin/python3
+
+# Update the module information used to generate related device IDs
+
+import json
+import os.path
+import subprocess
+import sys
+
+
+def iter_modules(base_dir):
+    def onerror(e):
+        raise e
+    for root, dirs, files in \
+        os.walk(os.path.join(base_dir, 'kernel'), onerror=onerror):
+        for name in files:
+            if name.endswith('.ko'):
+                yield name[:-3], os.path.join(root, name)
+
+
+def get_module_info(filename, attr_name):
+    output = subprocess.check_output(['modinfo', '-F', attr_name, filename],
+                                     text=True)
+    if output == '':
+        return []
+    return output.rstrip('\n').split('\n')
+
+
+def main(kernel_id=None):
+    if kernel_id is None:
+        kernel_dir = '/lib/modules/' + os.uname().release
+    elif '/' not in kernel_id:
+        kernel_dir = '/lib/modules/' + kernel_id
+    else:
+        kernel_dir = kernel_id
+
+    modinfo = {}
+    for name, filename in iter_modules(kernel_dir):
+        # We only care about modules that might request firmware
+        firmware = get_module_info(filename, 'firmware')
+        if not firmware:
+            continue
+
+        # We only care about aliases generated from device IDs, which
+        # start with <type> ":"
+        aliases = [alias
+                   for alias in get_module_info(filename, 'alias')
+                   if ':' in alias]
+
+        modinfo[name] = {
+            'alias': aliases,
+            'firmware': firmware,
+        }
+
+    with open('debian/modinfo.json', 'w') as f:
+        json.dump(modinfo, f, indent=2, sort_keys=True)
+
+
+if __name__ == '__main__':
+    main(*sys.argv[1:])

debian/changelog

 firmware-free (20200122-2) UNRELEASED; urgency=medium
 
+  [ Ben Hutchings ]
   * d/control: Use my debian.org email in Uploaders field
-  * Update to linux-support 5.15.0-3
   * d/salsa-ci.yml: Add CI configuration for salsa.debian.org
     - d/rules: Add linux-support-name target to show the name of the
       linux-support dependency
     - d/bin/gbp-postexport-hook: Add gbp hook to run gencontrol.py etc.
+  * d/b/gencontrol.py: Sync with firmware-nonfree
+  * Add debian/README.source based on the text for firmware-nonfree
+  * d/t/control.binary.in: Add Breaks, Provides, and Recommends fields
+  * d/l/p/config.py: Separate schemas for top-level and package config
+  * debian/bin: Add update-modinfo script to maintain cache of module info
+  * Add module info from linux 6.4.0-3 on amd64
+  * debian/bin/gencontrol.py: Include modaliases in AppStream metadata
+  * d/bin/gencontrol.py: Make debian/rules.gen reproducible
+  * d/rules, d/rules.real: Sync with firmware-nonfree
+  * Update to linux-support 6.4.0-3
+
+  [ Cyril Brulebois ]
+  * Update to the new linux-support API:
+     - Replace self.templates[foo] with self.templates.get*(foo),
+       merging the self.process_package() substitution in passing.
+     - Replace makefile.add() with makefile.add_cmds().
+     - Make Template inherit from _ControlFileDict, and use OrderedDict to
+       define its fields. Use PackageDescriptionBase directly for the
+       Description.
+     - Simplify Templates, only adding a get_templates_control() method to
+       the base class.
+     - Use get_templates_control() and an append() to include the license
+       in the templates.
+     - Avoid KeyError during substitutions by renaming @?variable@ into
+       @variable@, listing each such variable as optional (so that the
+       placeholder gets replaced with an empty string if that variable is
+       not set in the config).
+
+  [ Diederik de Haas ]
+  * d/patches/gitignore: Add DEP-3 header
+  * d/README.source: Expand steps to regenerate debian/control
 
  -- Ben Hutchings <benh@debian.org>  Sat, 22 Jan 2022 18:57:03 +0100
 

debian/lib/python/config.py

@@ -3,19 +3,24 @@ from debian_linux.config import ConfigParser, SchemaItemList
 class Config(dict):
     config_name = "defines"
 
-    schemas = {
+    top_schemas = {
         'base': {
-            'files': SchemaItemList(),
             'packages': SchemaItemList(),
-            'support': SchemaItemList(),
         },
     }
 
+    package_schemas = {
+        'base': {
+            'files': SchemaItemList(),
+            'support': SchemaItemList(),
+        }
+    }
+
     def __init__(self):
         self._read_base()
 
     def _read_base(self):
-        config = ConfigParser(self.schemas)
+        config = ConfigParser(self.top_schemas)
         config.read("debian/config/%s" % self.config_name)
 
         packages = config['base',]['packages']
@@ -28,7 +33,7 @@ class Config(dict):
             self._read_package(package)
 
     def _read_package(self, package):
-        config = ConfigParser(self.schemas)
+        config = ConfigParser(self.package_schemas)
         config.read("debian/config/%s/%s" % (package, self.config_name))
 
         for section in iter(config):

debian/patches/gitignore.patch

+From: Ben Hutchings <ben@decadent.org.uk>
+Date: Mon, 18 Oct 2015 03:29:18 +0200
+Subject: Git ignore everything except the debian dir
+Forwarded: not-needed
+
 --- /dev/null
 +++ b/.gitignore
 @@ -0,0 +1,6 @@

debian/rules

@@ -27,7 +27,7 @@ binary-arch: build-arch
 binary:	binary-indep binary-arch
 
 CONTROL_FILES = debian/build/version-info $(wildcard debian/templates/*.in)
-CONTROL_FILES += debian/bin/gencontrol.py debian/config/defines $(wildcard debian/config/*/defines) $(wildcard debian/config/*/copyright) $(wildcard debian/config/*/LICENSE*)
+CONTROL_FILES += debian/bin/gencontrol.py debian/config/defines $(wildcard debian/config/*/defines) debian/modinfo.json
 
 # debian/bin/gencontrol.py uses debian/changelog as input, but the
 # output only depends on the source name and version.  To avoid
@@ -75,7 +75,6 @@ endif
 maintainerclean:
 	-rm debian/control debian/control.md5sum debian/rules.gen
 	rm -f debian/*.bug-presubj
-	-rm debian/*.copyright
 	-rm debian/*.hook.*
 	rm -f debian/*.metainfo.xml
 	-rm debian/*.preinst

debian/rules.defs

-KERNELVERSION := 5.15.0-3
+KERNELVERSION := 6.4.0-3

debian/rules.real

@@ -2,8 +2,6 @@ SHELL  := sh -e
 
 export DH_OPTIONS
 
-include debian/rules.defs
-
 #
 # Targets
 #
@@ -24,6 +22,7 @@ install:
 	@for i in $(LINKS); do \
 	  link=debian/$(PACKAGE_NAME)/lib/firmware/"$${i%:*}"; \
 	  target="$${i#*:}"; \
+	  install -d "$${link%/*}"; \
 	  echo ln -s "$$target" "$$link"; \
 	  ln -s "$$target" "$$link"; \
 	done
@@ -33,8 +32,9 @@ ifneq ($(FILES),)
 endif
 	dh_bugfiles
 	dh_installchangelogs
-	dh_installdocs
+	dh_installdocs -XTODO
 	dh_installdebconf
+	dh_lintian
 	dh_link
 	dh_compress
 	dh_fixperms

debian/templates/control.binary.in

 Package: firmware-@package@
 Architecture: all
-Replaces: @?replaces@
-Conflicts: @?conflicts@
+Replaces: @replaces@
+Conflicts: @conflicts@
+Breaks: @breaks@
+Provides: @provides@
 Depends: ${misc:Depends}
+Recommends: @recommends@
 Suggests: initramfs-tools
 Description: Binary firmware for @desc@
  @longdesc@

debian/templates/metainfo.xml.in

@@ -5,6 +5,6 @@
   <summary>Binary firmware for @longdesc-metainfo@.</summary>
   <metadata_license>CC0-1.0</metadata_license>
   <provides>
-@firmware-list@
+@firmware-list@@modalias-list@
   </provides>
 </component>

debian/templates/metainfo.xml.modalias.in

+    <modalias>@alias@</modalias>