diffoscope/comparators/debian.py

@@ -336,7 +336,7 @@ class DotBuildinfoFile(DebControlFile):
     CONTAINER_CLASSES = [DotBuildinfoContainer]
     FILE_EXTENSION_SUFFIX = {".buildinfo"}
     FILE_TYPE_RE = re.compile(
-        r"^(ASCII text|UTF-8 Unicode text|PGP signed message)"
+        r"^(ASCII text|UTF-8 Unicode text|Unicode text, UTF-8 text|PGP signed message)"
     )
 
     @classmethod

tests/comparators/test_debian.py

@@ -26,7 +26,7 @@ from diffoscope.comparators.binary import FilesystemFile
 from diffoscope.comparators.missing_file import MissingFile
 from diffoscope.comparators.utils.specialize import specialize
 
-from ..utils.data import data, get_data
+from ..utils.data import data, get_data, load_fixture
 from ..utils.tools import skip_unless_module_exists
 
 from ..utils.nonexisting import assert_non_existing
@@ -44,24 +44,26 @@ except ImportError:
         DotBuildinfoFile,
     )
 
-TEST_DOT_CHANGES_FILE1_PATH = data("test1.changes")
-TEST_DOT_CHANGES_FILE2_PATH = data("test2.changes")
-TEST_DOT_CHANGES_FILE3_PATH = data("test3.changes")
-TEST_DOT_CHANGES_FILE4_PATH = data("test4.changes")
-TEST_DOT_BUILDINFO_FILE1_PATH = data("test1.buildinfo")
-TEST_DOT_BUILDINFO_FILE2_PATH = data("test2.buildinfo")
-TEST_DEB_FILE1_PATH = data("test1.deb")
-TEST_DEB_FILE2_PATH = data("test2.deb")
+TEST_DOT_CHANGES_FILE1 = data("test1.changes")
+TEST_DOT_CHANGES_FILE2 = data("test2.changes")
+TEST_DOT_CHANGES_FILE3 = data("test3.changes")
+TEST_DOT_CHANGES_FILE4 = data("test4.changes")
+TEST_DOT_BUILDINFO_FILE1 = data("test1.buildinfo")
+TEST_DOT_BUILDINFO_FILE2 = data("test2.buildinfo")
+TEST_DEB_FILE1 = data("test1.deb")
+TEST_DEB_FILE2 = data("test2.deb")
+
+changes5 = load_fixture("test5.changes")
 
 
 @pytest.fixture
 def dot_changes1(tmpdir):
     tmpdir.mkdir("a")
     dot_changes_path = str(tmpdir.join("a/test_1.changes"))
-    shutil.copy(TEST_DOT_CHANGES_FILE1_PATH, dot_changes_path)
-    shutil.copy(TEST_DEB_FILE1_PATH, str(tmpdir.join("a/test_1_all.deb")))
+    shutil.copy(TEST_DOT_CHANGES_FILE1, dot_changes_path)
+    shutil.copy(TEST_DEB_FILE1, str(tmpdir.join("a/test_1_all.deb")))
     shutil.copy(
-        TEST_DOT_BUILDINFO_FILE1_PATH, str(tmpdir.join("a/test_1.buildinfo"))
+        TEST_DOT_BUILDINFO_FILE1, str(tmpdir.join("a/test_1.buildinfo"))
     )
     return specialize(FilesystemFile(dot_changes_path))
 
@@ -70,10 +72,10 @@ def dot_changes1(tmpdir):
 def dot_changes2(tmpdir):
     tmpdir.mkdir("b")
     dot_changes_path = str(tmpdir.join("b/test_1.changes"))
-    shutil.copy(TEST_DOT_CHANGES_FILE2_PATH, dot_changes_path)
-    shutil.copy(TEST_DEB_FILE2_PATH, str(tmpdir.join("b/test_1_all.deb")))
+    shutil.copy(TEST_DOT_CHANGES_FILE2, dot_changes_path)
+    shutil.copy(TEST_DEB_FILE2, str(tmpdir.join("b/test_1_all.deb")))
     shutil.copy(
-        TEST_DOT_BUILDINFO_FILE2_PATH, str(tmpdir.join("b/test_2.buildinfo"))
+        TEST_DOT_BUILDINFO_FILE2, str(tmpdir.join("b/test_2.buildinfo"))
     )
     return specialize(FilesystemFile(dot_changes_path))
 
@@ -82,10 +84,10 @@ def dot_changes2(tmpdir):
 def dot_changes3(tmpdir):
     tmpdir.mkdir("c")
     dot_changes_path = str(tmpdir.join("c/test_3.changes"))
-    shutil.copy(TEST_DOT_CHANGES_FILE3_PATH, dot_changes_path)
-    shutil.copy(TEST_DEB_FILE1_PATH, str(tmpdir.join("c/test_1_all.deb")))
+    shutil.copy(TEST_DOT_CHANGES_FILE3, dot_changes_path)
+    shutil.copy(TEST_DEB_FILE1, str(tmpdir.join("c/test_1_all.deb")))
     shutil.copy(
-        TEST_DOT_BUILDINFO_FILE2_PATH, str(tmpdir.join("c/test_2.buildinfo"))
+        TEST_DOT_BUILDINFO_FILE2, str(tmpdir.join("c/test_2.buildinfo"))
     )
     return specialize(FilesystemFile(dot_changes_path))
 
@@ -94,10 +96,10 @@ def dot_changes3(tmpdir):
 def dot_changes4(tmpdir):
     tmpdir.mkdir("d")
     dot_changes_path = str(tmpdir.join("d/test_4.changes"))
-    shutil.copy(TEST_DOT_CHANGES_FILE4_PATH, dot_changes_path)
-    shutil.copy(TEST_DEB_FILE2_PATH, str(tmpdir.join("d/test_1_all.deb")))
+    shutil.copy(TEST_DOT_CHANGES_FILE4, dot_changes_path)
+    shutil.copy(TEST_DEB_FILE2, str(tmpdir.join("d/test_1_all.deb")))
     shutil.copy(
-        TEST_DOT_BUILDINFO_FILE1_PATH, str(tmpdir.join("d/test_2.buildinfo"))
+        TEST_DOT_BUILDINFO_FILE1, str(tmpdir.join("d/test_2.buildinfo"))
     )
     return specialize(FilesystemFile(dot_changes_path))
 
@@ -110,7 +112,7 @@ def test_dot_changes_identification(dot_changes1):
 def test_dot_changes_invalid(tmpdir):
     tmpdir.mkdir("a")
     dot_changes_path = str(tmpdir.join("a/test_1.changes"))
-    shutil.copy(TEST_DOT_CHANGES_FILE1_PATH, dot_changes_path)
+    shutil.copy(TEST_DOT_CHANGES_FILE1, dot_changes_path)
     # we don't copy the referenced .deb
     identified = specialize(FilesystemFile(dot_changes_path))
     # ... but it is identified regardless
@@ -201,18 +203,18 @@ def test_dot_changes_different_contents_and_identical_files(
     )
 
 
-TEST_DOT_DSC_FILE1_PATH = data("test1.dsc")
-TEST_DOT_DSC_FILE2_PATH = data("test2.dsc")
-TEST_DEB_SRC1_PATH = data("test1.debsrc.tar.gz")
-TEST_DEB_SRC2_PATH = data("test2.debsrc.tar.gz")
+TEST_DOT_DSC_FILE1 = data("test1.dsc")
+TEST_DOT_DSC_FILE2 = data("test2.dsc")
+TEST_DEB_SRC1 = data("test1.debsrc.tar.gz")
+TEST_DEB_SRC2 = data("test2.debsrc.tar.gz")
 
 
 @pytest.fixture
 def dot_dsc1(tmpdir):
     tmpdir.mkdir("a")
     dot_dsc_path = str(tmpdir.join("a/test_1.dsc"))
-    shutil.copy(TEST_DOT_DSC_FILE1_PATH, dot_dsc_path)
-    shutil.copy(TEST_DEB_SRC1_PATH, str(tmpdir.join("a/test_1.tar.gz")))
+    shutil.copy(TEST_DOT_DSC_FILE1, dot_dsc_path)
+    shutil.copy(TEST_DEB_SRC1, str(tmpdir.join("a/test_1.tar.gz")))
     return specialize(FilesystemFile(dot_dsc_path))
 
 
@@ -220,8 +222,8 @@ def dot_dsc1(tmpdir):
 def dot_dsc2(tmpdir):
     tmpdir.mkdir("b")
     dot_dsc_path = str(tmpdir.join("b/test_1.dsc"))
-    shutil.copy(TEST_DOT_DSC_FILE2_PATH, dot_dsc_path)
-    shutil.copy(TEST_DEB_SRC2_PATH, str(tmpdir.join("b/test_1.tar.gz")))
+    shutil.copy(TEST_DOT_DSC_FILE2, dot_dsc_path)
+    shutil.copy(TEST_DEB_SRC2, str(tmpdir.join("b/test_1.tar.gz")))
     return specialize(FilesystemFile(dot_dsc_path))
 
 
@@ -233,7 +235,7 @@ def test_dot_dsc_identification(dot_dsc1):
 def test_dot_dsc_no_associated_tar_gz(tmpdir, dot_dsc2):
     tmpdir.mkdir("a")
     dot_dsc_path = str(tmpdir.join("a/test_1.dsc"))
-    shutil.copy(TEST_DOT_CHANGES_FILE1_PATH, dot_dsc_path)
+    shutil.copy(TEST_DOT_CHANGES_FILE1, dot_dsc_path)
     # we don't copy the referenced .tar.gz
     identified = specialize(FilesystemFile(dot_dsc_path))
     assert isinstance(identified, DotDscFile)
@@ -267,9 +269,9 @@ def test_dot_dsc_compare_non_existing(monkeypatch, dot_dsc1):
 def dot_buildinfo1(tmpdir):
     tmpdir.mkdir("a")
     dot_buildinfo_path = str(tmpdir.join("a/test_1.buildinfo"))
-    shutil.copy(TEST_DOT_BUILDINFO_FILE1_PATH, dot_buildinfo_path)
-    shutil.copy(TEST_DOT_DSC_FILE1_PATH, str(tmpdir.join("a/test_1.dsc")))
-    shutil.copy(TEST_DEB_FILE1_PATH, str(tmpdir.join("a/test_1_all.deb")))
+    shutil.copy(TEST_DOT_BUILDINFO_FILE1, dot_buildinfo_path)
+    shutil.copy(TEST_DOT_DSC_FILE1, str(tmpdir.join("a/test_1.dsc")))
+    shutil.copy(TEST_DEB_FILE1, str(tmpdir.join("a/test_1_all.deb")))
     return specialize(FilesystemFile(dot_buildinfo_path))
 
 
@@ -277,9 +279,9 @@ def dot_buildinfo1(tmpdir):
 def dot_buildinfo2(tmpdir):
     tmpdir.mkdir("b")
     dot_buildinfo_path = str(tmpdir.join("b/test_1.buildinfo"))
-    shutil.copy(TEST_DOT_BUILDINFO_FILE2_PATH, dot_buildinfo_path)
-    shutil.copy(TEST_DOT_DSC_FILE2_PATH, str(tmpdir.join("b/test_1.dsc")))
-    shutil.copy(TEST_DEB_FILE2_PATH, str(tmpdir.join("b/test_1_all.deb")))
+    shutil.copy(TEST_DOT_BUILDINFO_FILE2, dot_buildinfo_path)
+    shutil.copy(TEST_DOT_DSC_FILE2, str(tmpdir.join("b/test_1.dsc")))
+    shutil.copy(TEST_DEB_FILE2, str(tmpdir.join("b/test_1_all.deb")))
     return specialize(FilesystemFile(dot_buildinfo_path))
 
 
@@ -291,7 +293,7 @@ def test_dot_buildinfo_identification(dot_buildinfo1):
 def test_dot_buildinfo_no_deb(tmpdir):
     tmpdir.mkdir("a")
     dot_buildinfo_path = str(tmpdir.join("a/test_1.buildinfo"))
-    shutil.copy(TEST_DOT_BUILDINFO_FILE1_PATH, dot_buildinfo_path)
+    shutil.copy(TEST_DOT_BUILDINFO_FILE1, dot_buildinfo_path)
     # we don't copy the referenced .deb
     identified = specialize(FilesystemFile(dot_buildinfo_path))
     assert isinstance(identified, DotBuildinfoFile)
@@ -351,3 +353,13 @@ def test_fallback_comparisons(monkeypatch):
         assert file1.compare(file1) is None
         assert file2.compare(file2) is None
         assert file1.compare(file2).unified_diff == get_data(expected_diff)
+
+
+def test_unicode_identification(changes5):
+    # .changes can be identified by file(1) as:
+    #
+    # * "ASCII text"
+    # * "UTF-8 Unicode text" (older versions of file)
+    # * "Unicode text, UTF-8 text"
+    # * "data" (files with broken Unicode: reproducible-builds/diffoscope#286)
+    assert isinstance(changes5, DotChangesFile)

tests/data/test5.changes

+Format: 1.8
+Date: Sat, 04 Apr 2015 18:30:48 +0200
+Source: test
+Binary: test
+Architecture: source all
+Version: 1
+Distribution: unstable
+Urgency: low
+Maintainer: Someone Else 𝐀 𝐁 𝐂 <user@example.org>
+Changed-By: Someone Else 𝐀 𝐁 𝐂 <user@example.org>
+Description:
+ test - just a simple test package
+Changes:
+ test (1) unstable; urgency=low
+ .
+   * Test package.
+   * Unicode test: 𝐀 𝐁 𝐂 𝐃 𝐄 𝐅 𝐆 𝐇 𝐈 𝐉 𝐊 𝐋 𝐌 𝐍 𝐎 𝐏 𝐐 𝐑 𝐒 𝐓 𝐔 𝐕 𝐖 𝐗 𝐘 𝐙.
+Checksums-Sha1:
+ 70982664db2015334bff6441b429d7e3c58dbecb 2388 test_1_all.deb
+ 91d2cc6aadddb4a24b0c2f8f24d558ce0e07f9cd 3765 test_2.buildinfo
+Checksums-Sha256:
+ 2f2e45ee3a5fdacd9b30133ec728121588bf9b97af3b947b3882b2b28a0555da 2388 test_1_all.deb
+ 917d8b5c3ade2bde26fe2b7476481537f32a0c07f10e5e44c0ca7a02c8bfa9d8 3765 test_2.buildinfo
+Files:
+ d323c454462407fe3bfde31a74b23eba 2388 devel optional test_1_all.deb
+ 9972d7c394228311ba92cbcbfe2d6cd9 3765 web optional test_2.buildinfo

tests/test_source.py

@@ -218,6 +218,7 @@ ALLOWED_TEST_FILES = {
     "test4.changes",
     "test4.gif",
     "test4.pdf",
+    "test5.changes",
     "test_comment1.zip",
     "test_comment2.zip",
     "test_invalid.json",