debian/control

@@ -83,6 +83,7 @@ Build-Depends:
  wabt <!nocheck>,
  xmlbeans <!nocheck>,
  xxd <!nocheck> | vim-common <!nocheck>,
+ zstd <!nocheck>,
 Build-Conflicts:
  graphicsmagick-imagemagick-compat,
 Standards-Version: 4.4.1

diffoscope/comparators/__init__.py

@@ -105,6 +105,7 @@ class ComparatorManager:
         ('ogg.OggFile',),
         ('xsb.XsbFile',),
         ('berkeley_db.BerkeleyDBFile',),
+        ('zst.ZstFile',),
     )
 
     _singleton = {}

diffoscope/comparators/zst.py

+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2019 Jelle van der Waa <jelle@archlinux.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+import os.path
+import logging
+import subprocess
+
+from diffoscope.tools import tool_required
+
+from .utils.file import File
+from .utils.archive import Archive
+
+logger = logging.getLogger(__name__)
+
+
+class ZstContainer(Archive):
+    def open_archive(self):
+        return self
+
+    def close_archive(self):
+        pass
+
+    def get_member_names(self):
+        return [self.get_compressed_content_name('.zst')]
+
+    @tool_required('zstd')
+    def extract(self, member_name, dest_dir):
+        dest_path = os.path.join(dest_dir, member_name)
+        logger.debug('zstd extracting to %s', dest_path)
+        with open(dest_path, 'wb') as fp:
+            subprocess.check_call(
+                ["zstd", "-d", "-c", self.source.path],
+                shell=False,
+                stdout=fp,
+                stderr=None,
+            )
+        return dest_path
+
+
+class ZstFile(File):
+    DESCRIPTION = "Zstandard compressed files"
+    CONTAINER_CLASS = ZstContainer
+    FILE_TYPE_RE = re.compile(r'^Zstandard compressed data')

diffoscope/external_tools.py

@@ -212,6 +212,7 @@ EXTERNAL_TOOLS = {
     'zipnote': {'debian': 'zip', 'guix': 'zip'},
     'procyon': {'debian': 'procyon-decompiler'},
     'dumpxsb': {'debian': 'xmlbeans'},
+    'zstd': {'debian': 'zstd'},
 }
 
 # May be populated at runtime by remapped names like

tests/comparators/test_ocaml.py

@@ -57,7 +57,7 @@ def ocaml_version():
     return out.decode('utf-8').split()[-1]
 
 
-@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
 def test_identification(cmi1):
     assert isinstance(cmi1, OcamlInterfaceFile)
 
@@ -67,18 +67,18 @@ def differences(cmi1, cmi2):
     return cmi1.compare(cmi2).details
 
 
-@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
 def test_no_differences(cmi1):
     difference = cmi1.compare(cmi1)
     assert difference is None
 
 
-@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
 def test_diff(differences):
     expected_diff = get_data('ocaml_expected_diff')
     assert differences[0].unified_diff == expected_diff
 
 
-@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+@skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
 def test_compare_non_existing(monkeypatch, cmi1):
     assert_non_existing(monkeypatch, cmi1, has_null_source=False)

tests/comparators/test_pcap.py

@@ -46,7 +46,7 @@ def differences(pcap1, pcap2):
 @skip_unless_tools_exist('tcpdump')
 def test_diff(differences):
     expected_diff = get_data('pcap_expected_diff')
-    assert differences[0].unified_diff == expected_diff
+    assert differences[0].unified_diff[: 2 ** 13] == expected_diff[: 2 ** 13]
 
 
 @skip_unless_tools_exist('tcpdump')

tests/comparators/test_zst.py

+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2019 Jelle van der Waa <jelle@archlinux.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
+
+import subprocess
+
+import pytest
+
+from diffoscope.comparators.zst import ZstFile
+from diffoscope.comparators.binary import FilesystemFile
+from diffoscope.comparators.utils.specialize import specialize
+
+from ..utils.tools import skip_unless_tools_exist
+
+
+def zst_fixture(prefix):
+    @pytest.fixture
+    def zstd(tmpdir):
+        input_ = str(tmpdir.join('{}'.format(prefix)))
+        output = str(tmpdir.join('{}.zst'.format(prefix)))
+
+        with open(input_, 'w') as f:
+            f.write(prefix)
+
+        subprocess.check_call(('zstd', input_))
+
+        return specialize(FilesystemFile(output))
+
+    return zstd
+
+
+zst1 = zst_fixture('test1')
+zst2 = zst_fixture('test2')
+
+
+@skip_unless_tools_exist('zstd')
+def test_identification(zst1):
+    assert isinstance(zst1, ZstFile)
+
+
+@skip_unless_tools_exist('zstd')
+def test_no_differences(zst1):
+    difference = zst1.compare(zst1)
+    assert difference is None
+
+
+@pytest.fixture
+def differences(zst1, zst2):
+    return zst1.compare(zst2).details
+
+
+@skip_unless_tools_exist('zstd')
+def test_content_source(differences):
+    assert differences[0].source1 == 'test1'
+    assert differences[0].source2 == 'test2'

tests/data/ocaml_expected_diff

@@ -2,7 +2,7 @@
 -Unit name: Test1
 +Unit name: Test2
  Interfaces imported:
--	b5f47409a5d1c32a864ea3d2fbed44db	Test1
-+	9ee19396ca59dbe57206a9a88d98a7d5	Test2
- 	07ea9e20ae94d62c35cfecbe7d66d3ea	Pervasives
- 	cbd5f2d6b649925222e1e9fb63b89db6	CamlinternalFormatBasics
+-	4bf3070814d7fb8e8d365d95481f8cad	Test1
++	333f54d1aae1264e7ad64cbb437cbc4f	Test2
+ 	ad45f251bbf98d3a0bf3b883546ecfc8	Stdlib
+ 	a2b1a9d869fd05813beb35645bd9cd94	CamlinternalFormatBasics

tests/test_source.py

@@ -18,12 +18,224 @@
 # along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
 
 import os
+import glob
 import diffoscope
 import subprocess
 
 from .utils.tools import skip_unless_tools_exist
 
-BASE_DIR = os.path.dirname(os.path.abspath(diffoscope.__file__))
+ALLOWED_TEST_FILES = {
+    # Data files we would prefer to generate dynamically
+    'android1.img',
+    'android2.img',
+    'archive1.tar',
+    'archive2.tar',
+    'base-files_157-r45695_ar71xx.ipk',
+    'base-files_157-r45918_ar71xx.ipk',
+    'binary1',
+    'binary2',
+    'bug881937_1.deb',
+    'bug881937_2.deb',
+    'bug903391_1.deb',
+    'bug903391_2.deb',
+    'bug903401_1.deb',
+    'bug903401_2.deb',
+    'bug903565_1.deb',
+    'bug903565_2.deb',
+    'containers',
+    'containers/a.tar.bz2',
+    'containers/a.tar.gz',
+    'containers/a.tar.xz',
+    'containers/b.tar.bz2',
+    'containers/b.tar.gz',
+    'containers/b.tar.xz',
+    'containers/magic_bzip2',
+    'containers/magic_gzip',
+    'containers/magic_xz',
+    'dbgsym/add/test-dbgsym-dbgsym_1_amd64.deb',
+    'dbgsym/add/test-dbgsym_1_amd64.deb',
+    'dbgsym/mult/test-dbgsym-dbgsym_1_amd64.deb',
+    'dbgsym/mult/test-dbgsym_1_amd64.deb',
+    'dbgsym/test-dbgsym_1.dsc',
+    'dbgsym/test-dbgsym_1.tar.gz',
+    'debian-bug-876316-control.tar.gz',
+    # Outputs
+    'devicetree1.dtb',
+    'devicetree2.dtb',
+    'elfmix1.not_a',
+    'elfmix2.a',
+    'encrypted1.zip',
+    'encrypted2.zip',
+    'fuzzy-tar-in-tar1.tar',
+    'fuzzy-tar-in-tar2.tar',
+    'fuzzy1.tar',
+    'fuzzy2.tar',
+    'fuzzy3.tar',
+    'hello1.wasm',
+    'hello2.wasm',
+    'no-perms.tar',
+    'quine.gz',
+    'quine.zip',
+    'quine_a.deb',
+    'quine_b.deb',
+    'Samyak-Malayalam1.ttf',
+    'Samyak-Malayalam2.ttf',
+    'test1-le64.cache-4',
+    'test1.a',
+    'test1.apk',
+    'test1.asc',
+    'test1.binwalk',
+    'test1.buildinfo',
+    'test1.bz2',
+    'test1.changes',
+    'Test1.class',
+    'Test2.class',
+    'test1.cpio',
+    'test1.db',
+    'test1.deb',
+    'test1.debsrc.tar.gz',
+    'test1.debug',
+    'test1.dex',
+    'test1.docx',
+    'test1.dsc',
+    'test1.epub',
+    'test1.exe',
+    'test1.ext4',
+    'test1.fat12',
+    'test1.fat16',
+    'test1.fat32',
+    'test1.gif',
+    'test1.git-index',
+    'test1.gnumeric',
+    'test1.gz',
+    'test1.hi',
+    'test1.icc',
+    'test1.ico',
+    'test1.iso',
+    'test1.jmod',
+    'test1.jpg',
+    'test1.js',
+    'test1.json',
+    'test1.kbx',
+    'test1.lz4',
+    'test1.macho',
+    'test1.mo',
+    'test1.mozzip',
+    'test1.mp3',
+    'test1.o',
+    'test1.odt',
+    'test1.ogg',
+    'test1.pcap',
+    'test1.pdf',
+    'test1.pgp',
+    'test1.png',
+    'test1.ppu',
+    'test1.ps',
+    'test1.rdx',
+    'test1.rlib',
+    'test1.rpm',
+    'test1.sqlite3',
+    'test1.squashfs',
+    'test1.tar',
+    'test1.xml',
+    'test1.xsb',
+    'test1.xz',
+    'test1.zip',
+    'test1_meta.ico',
+    'test1_meta.jpg',
+    'test2-le64.cache-4',
+    'test2.a',
+    'test2.apk',
+    'test2.asc',
+    'test2.binwalk',
+    'test2.buildinfo',
+    'test2.bz2',
+    'test2.changes',
+    'test2.cpio',
+    'test2.db',
+    'test2.deb',
+    'test2.debsrc.tar.gz',
+    'test2.debug',
+    'test2.dex',
+    'test2.docx',
+    'test2.dsc',
+    'test2.epub',
+    'test2.exe',
+    'test2.ext4',
+    'test2.fat12',
+    'test2.gif',
+    'test2.git-index',
+    'test2.gnumeric',
+    'test2.gz',
+    'test2.hi',
+    'test2.icc',
+    'test2.ico',
+    'test2.iso',
+    'test2.jmod',
+    'test2.jpg',
+    'test2.js',
+    'test2.json',
+    'test2.kbx',
+    'test2.lz4',
+    'test2.macho',
+    'test2.mo',
+    'test2.mozzip',
+    'test2.mp3',
+    'test2.o',
+    'test2.odt',
+    'test2.ogg',
+    'test2.pcap',
+    'test2.pdf',
+    'test2.pgp',
+    'test2.png',
+    'test2.ppu',
+    'test2.ps',
+    'test2.rdx',
+    'test2.rlib',
+    'test2.rpm',
+    'test2.sqlite3',
+    'test2.squashfs',
+    'test2.tar',
+    'test2.xml',
+    'test2.xsb',
+    'test2.xz',
+    'test2.zip',
+    'test2_meta.ico',
+    'test2_meta.jpg',
+    'test3.apk',
+    'test3.changes',
+    'test3.gif',
+    'test3.zip',
+    'test4.changes',
+    'test4.gif',
+    'test_comment1.zip',
+    'test_comment2.zip',
+    'test_invalid.json',
+    'test_invalid.xml',
+    'test_iso8859-1.mo',
+    'test_no_charset.mo',
+    'test_openssh_pub_key1.pub',
+    'test_openssh_pub_key2.pub',
+    'test_weird_non_unicode_chars1.pdf',
+    'test_weird_non_unicode_chars2.pdf',
+    'text_ascii1',
+    'text_ascii2',
+    'text_iso8859',
+    'text_order1',
+    'text_order2',
+    'text_unicode1',
+    'text_unicode2',
+    'text_unicode_binary_fallback',
+    # Outputs
+    'debian-bug-875281.collapsed-diff.json',
+    'order1a.json',
+    'order1b.json',
+    'output.colored.txt',
+    'output.json',
+    'output.md',
+    'output.rst',
+    'output.txt',
+}
 
 
 @skip_unless_tools_exist('black')
@@ -36,3 +248,40 @@ def test_code_is_black_clean():
     print(output)
 
     assert len(output) == 0
+
+
+def test_does_not_add_new_test_files():
+    """
+    For a variety of reasons we are now prefering to generate any test data
+    dynamically (via pytest fixtures, etc.) rather than committing and shipping
+    such files.
+
+    Exceptions to this may be appropriate (or even required) but this test
+    ensures that test files that could be dynamically generated are not added
+    "automatically", for example by following previous/older commits.
+    """
+
+    test_dir = os.path.join(os.path.dirname(__file__), 'data')
+
+    seen = set()
+
+    for x in glob.iglob(os.path.join(test_dir, '**'), recursive=True):
+        if os.path.isdir(x):
+            continue
+
+        # Strip off common prefix
+        x = x[len(test_dir) + 1 :]
+
+        # Skip some known expected diff filename patterns
+        if (
+            x.endswith('_diff')
+            or x.endswith('_diffs')
+            or x.endswith('.diff')
+            or '_diff_' in x
+            or 'diff.' in x
+        ):
+            continue
+
+        seen.add(x)
+
+    assert seen - ALLOWED_TEST_FILES - {''} == set()