Improve DOS/MBR extraction by adding support for 7z. (Closes: #333)

debian/tests/control

@@ -7,7 +7,7 @@
 #   $ mv debian/tests/control.tmp debian/tests/control
 
 Tests: pytest-with-recommends
-Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], aapt [amd64 arm64 armel armhf i386 mips64el mipsel], abootimg, acl, apksigcopier, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, dexdump [amd64 arm64 armhf i386], docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg, gnupg-utils, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-utils, llvm, lz4 | liblz4-tool, lzip, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd, androguard, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh
+Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], aapt [amd64 arm64 armel armhf i386 mips64el mipsel], abootimg, acl, apksigcopier, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, dexdump [amd64 arm64 armhf i386], docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg, gnupg-utils, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-utils, llvm, lz4 | liblz4-tool, lzip, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, p7zip-full, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd, androguard, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh
 
 Tests: pytest
 Depends: python3-all, diffoscope, python3-pytest, python3-h5py, file, python3-tlsh

diffoscope/comparators/7z.py

+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2023 Chris Lamb <lamby@debian.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
+
+import os
+import re
+import logging
+import subprocess
+
+from diffoscope.tools import tool_required
+from diffoscope.tempfiles import get_temporary_directory
+from diffoscope.difference import Difference
+
+from .utils.archive import Archive
+from .utils.file import File
+from .utils.command import Command, our_check_output
+
+logger = logging.getLogger(__name__)
+
+
+class SevenZList(Command):
+    @tool_required("7z")
+    def cmdline(self):
+        return (
+            "7z",
+            "l",
+            self.path,
+        )
+
+    def filter(self, line):
+        val = line.decode("utf-8")
+        if val.startswith("Listing archive: ") or val.startswith("Path = "):
+            return b""
+        return line
+
+
+class SevenZContainer(Archive):
+    @tool_required("7z")
+    def open_archive(self):
+        self._temp_dir = get_temporary_directory(suffix="7z")
+
+        try:
+            our_check_output(
+                ("7z", "e", os.path.abspath(self.source.path)),
+                cwd=self._temp_dir.name,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError:
+            return False
+
+        return self
+
+    def close_archive(self):
+        self._temp_dir.cleanup()
+
+    def get_member_names(self):
+        return os.listdir(self._temp_dir.name)
+
+    def extract(self, member_name, dest_dir):
+        return os.path.join(self._temp_dir.name, member_name)
+
+
+class SevenZFile(File):
+    DESCRIPTION = "Filesystem image"
+    FILE_TYPE_RE = re.compile(r"^DOS/MBR boot sector;")
+    CONTAINER_CLASSES = [SevenZContainer]
+
+    def compare_details(self, other, source=None):
+        return [
+            Difference.from_operation(
+                SevenZList, self.path, other.path, source="7z l"
+            )
+        ]
+
+    @classmethod
+    def recognizes(cls, file):
+        if not super().recognizes(file):
+            return False
+
+        try:
+            cmd = SevenZList(file.path)
+            cmd.start()
+        except RequiredToolNotFound:
+            return False
+
+        return b"Type = gzip\n" not in cmd.output

diffoscope/comparators/__init__.py

@@ -75,6 +75,7 @@ class ComparatorManager:
         ("ffprobe.FfprobeFile",),
         ("gnumeric.GnumericFile",),
         ("gzip.GzipFile",),
+        ("7z.SevenZFile",),
         ("haskell.HiFile",),
         ("icc.IccFile",),
         ("iso9660.Iso9660File",),

diffoscope/external_tools.py

@@ -23,6 +23,7 @@ that might resolve to, for example, `/usr/bin/abootimg`.
 """
 
 EXTERNAL_TOOLS = {
+    "7z": {"debian": "p7zip-full"},
     "aapt2": {"debian": "aapt"},
     "abootimg": {"debian": "abootimg", "guix": "abootimg"},
     "androguard": {"debian": "androguard"},

tests/comparators/test_7z.py

+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2023 Chris Lamb <lamby@debian.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
+
+import shutil
+import pytest
+
+from diffoscope.comparators.lz4 import Lz4File
+from diffoscope.comparators.binary import FilesystemFile
+from diffoscope.comparators.utils.specialize import specialize
+
+from ..utils.data import load_fixture, assert_diff
+from ..utils.tools import skip_unless_tools_exist
+from ..utils.nonexisting import assert_non_existing
+
+lz4a = load_fixture("test1.lz4")
+lz4b = load_fixture("test2.lz4")
+
+
+def test_identification(lz4a):
+    assert isinstance(lz4a, Lz4File)
+
+
+def test_no_differences(lz4a):
+    difference = lz4a.compare(lz4a)
+    assert difference is None
+
+
+@pytest.fixture
+def differences(lz4a, lz4b):
+    return lz4a.compare(lz4b).details
+
+
+@skip_unless_tools_exist("lz4")
+def test_content_source(differences):
+    assert differences[0].source1 == "test1"
+    assert differences[0].source2 == "test2"
+
+
+@skip_unless_tools_exist("lz4")
+def test_content_source_without_extension(tmpdir, lz4a, lz4b):
+    path1 = str(tmpdir.join("test1"))
+    path2 = str(tmpdir.join("test2"))
+    shutil.copy(lz4a.path, path1)
+    shutil.copy(lz4b.path, path2)
+    lz4a = specialize(FilesystemFile(path1))
+    lz4b = specialize(FilesystemFile(path2))
+    difference = lz4a.compare(lz4b).details
+    assert difference[0].source1 == "test1-content"
+    assert difference[0].source2 == "test2-content"
+
+
+@skip_unless_tools_exist("lz4")
+def test_content_diff(differences):
+    assert_diff(differences[0], "text_ascii_expected_diff")
+
+
+@skip_unless_tools_exist("lz4")
+def test_compare_non_existing(monkeypatch, lz4a):
+    assert_non_existing(monkeypatch, lz4a)