debian/copyright

@@ -27,7 +27,7 @@ Copyright:
  © 2018 Xavier Briand <xavierbriand@gmail.com>
  © 2019 Jelle van der Waa <jelle@archlinux.org>
  © 2020 Conrad Ratschan <ratschance@gmail.com>
- © 2020 Jean-Romain Garnier <salsa@jean-romain.com>
+ © 2020-2021 Jean-Romain Garnier <salsa@jean-romain.com>
 License: GPL-3+
 
 Files: diffoscope/changes.py

diffoscope/comparators/__init__.py

@@ -43,6 +43,8 @@ class ComparatorManager:
         ("deb.DebDataTarFile",),
         ("decompile.AsmFunction",),
         ("elf.ElfSection",),
+        ("macho.MachoSection",),
+        ("macho.MachoArchitecture",),
         ("binwalk.BinwalkFile",),
         ("ps.PsFile",),
         ("javascript.JavaScriptFile",),

diffoscope/external_tools.py

@@ -2,6 +2,7 @@
 # diffoscope: in-depth comparison of files, archives, and directories
 #
 # Copyright © 2017-2020 Chris Lamb <lamby@debian.org>
+# Copyright © 2021 Jean-Romain Garnier <salsa@jean-romain.com>
 #
 # diffoscope is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -112,6 +113,8 @@ EXTERNAL_TOOLS = {
     "llvm-bcanalyzer": {"debian": "llvm", "arch": "llvm", "guix": "llvm"},
     "llvm-config": {"debian": "llvm", "arch": "llvm"},
     "llvm-dis": {"debian": "llvm", "arch": "llvm", "guix": "llvm"},
+    "llvm-readobj": {"debian": "llvm", "arch": "llvm", "guix": "llvm"},
+    "llvm-objdump": {"debian": "llvm", "arch": "llvm", "guix": "llvm"},
     "ls": {"debian": "coreutils", "arch": "coreutils", "guix": "coreutils"},
     "lsattr": {
         "debian": "e2fsprogs",
@@ -256,6 +259,8 @@ HUGE_TOOLS = {
     "llvm-bcanalyzer",
     "llvm-config",
     "llvm-dis",
+    "llvm-readobj",
+    "llvm-objdump",
     "ppudump",
     "javap",
     "ssconvert",

tests/comparators/test_elf.py

@@ -3,6 +3,7 @@
 #
 # Copyright © 2015 Jérémy Bobbio <lunar@debian.org>
 # Copyright © 2015-2020 Chris Lamb <lamby@debian.org>
+# Copyright © 2021 Jean-Romain Garnier <salsa@jean-romain.com>
 #
 # diffoscope is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -169,6 +170,7 @@ def libmix_differences(libmix1, libmix2):
 
 
 @skip_unless_tools_exist("xxd")
+@skip_unless_tools_exist("llvm-readobj", "llvm-objdump")
 @skip_unless_tools_exist("readelf", "objdump")
 @skip_if_tool_version_is("readelf", readelf_version, "2.29")
 @skip_if_binutils_does_not_support_x86()
@@ -178,17 +180,23 @@ def test_libmix_differences(libmix_differences):
 
     # Check order and basic identification
     assert file_list.source1 == "file list"
-    assert "Falling back to binary" in mach_o.comments[0]
     x86_o = x86_o.details[0]
     assert x86_o.source1.startswith("objdump ")
     assert src_c.source1.endswith(".c")
 
+    mach_o = mach_o.details[0]
+    for diff in mach_o.details:
+        assert diff.source1.startswith("llvm-objdump ")
+
     # Content
     assert "return42_or_3" in file_list.unified_diff
-    assert_diff(mach_o, "elfmix_mach_o_expected_diff")
     assert_diff(x86_o, "elfmix_disassembly_expected_diff")
     assert_diff(src_c, "elfmix_src_c_expected_diff")
 
+    mach_o_filenames = ["elfmix_mach_o_expected_diff__text"]
+    for idx, diff in enumerate(mach_o.details):
+        assert_diff(diff, mach_o_filenames[idx])
+
     x_obj = x_obj.details[0]
     if x_obj.source1.startswith("readelf "):
         assert_diff(x_obj, "elfmix_x_obj_expected_diff")

tests/comparators/test_elf_decompiler.py

@@ -57,7 +57,7 @@ def skip_unless_radare2_command_exists(command):
 def exclude_commands(monkeypatch, patterns):
     excluded = list(Config().exclude_commands)
     excluded += patterns
-    monkeypatch.setattr(Config(), "exclude_commands", patterns)
+    monkeypatch.setattr(Config(), "exclude_commands", excluded)
 
 
 @pytest.fixture(scope="function", autouse=True)
@@ -89,7 +89,7 @@ def test_obj_compare_non_existing(monkeypatch, obj1):
 @skip_unless_module_exists("r2pipe")
 @skip_unless_radare2_command_exists("pdgj")
 def test_ghidra_diff(monkeypatch, obj1, obj2):
-    exclude_commands(monkeypatch, ["disass.*"])
+    exclude_commands(monkeypatch, ["^radare2 disass.*"])
     obj_differences = obj1.compare(obj2).details[0].details
     assert len(obj_differences) == 1
     assert_diff(obj_differences[0], "elf_obj_ghidra_expected_diff")
@@ -98,7 +98,7 @@ def test_ghidra_diff(monkeypatch, obj1, obj2):
 @skip_unless_tools_exist("radare2")
 @skip_unless_module_exists("r2pipe")
 def test_radare2_diff(monkeypatch, obj1, obj2):
-    exclude_commands(monkeypatch, ["r2ghidra.*"])
+    exclude_commands(monkeypatch, ["^radare2 r2ghidra.*"])
     obj_differences = obj1.compare(obj2).details[0].details
     assert len(obj_differences) == 1
     assert_diff(obj_differences[0], "elf_obj_radare2_expected_diff")

tests/comparators/test_macho.py

@@ -4,6 +4,7 @@
 # Copyright © 2015 Jérémy Bobbio <lunar@debian.org>
 # Copyright © 2015 Clemens Lang <cal@macports.org>
 # Copyright © 2016-2020 Chris Lamb <lamby@debian.org>
+# Copyright © 2021 Jean-Romain Garnier <salsa@jean-romain.com>
 #
 # diffoscope is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,7 +25,7 @@ from diffoscope.config import Config
 from diffoscope.comparators.macho import MachoFile
 from diffoscope.comparators.missing_file import MissingFile
 
-from ..utils.data import load_fixture, get_data
+from ..utils.data import load_fixture, assert_diff
 from ..utils.tools import skip_unless_tools_exist
 
 
@@ -32,6 +33,13 @@ obj1 = load_fixture("test1.macho")
 obj2 = load_fixture("test2.macho")
 
 
+@pytest.fixture(scope="function", autouse=True)
+def init_tests(request, monkeypatch):
+    # Ignore radare2 commands so decompiling is skipped
+    # See test_macho_decompiler.py for tests related to decompiler
+    monkeypatch.setattr(Config(), "exclude_commands", ["^radare2.*"])
+
+
 def test_obj_identification(obj1):
     assert isinstance(obj1, MachoFile)
 
@@ -47,7 +55,7 @@ def obj_differences(obj1, obj2):
 
 
 @skip_unless_tools_exist("otool", "lipo")
-def test_obj_compare_non_existing(monkeypatch, obj1):
+def test_otool_obj_compare_non_existing(monkeypatch, obj1):
     monkeypatch.setattr(Config(), "new_file", True)
     difference = obj1.compare(MissingFile("/nonexisting", obj1))
     assert difference.source2 == "/nonexisting"
@@ -55,13 +63,60 @@ def test_obj_compare_non_existing(monkeypatch, obj1):
 
 
 @skip_unless_tools_exist("otool", "lipo")
-def test_diff(obj_differences):
-    assert len(obj_differences) == 4
+def test_otool_diff(obj_differences):
+    assert len(obj_differences) == 2
+    assert_diff(obj_differences[0], "macho_otool_expected_diff_strings")
+
+    arch_differences = obj_differences[-1].details
+    assert len(arch_differences) == 6
+
+    filenames = [
+        "macho_otool_expected_diff_headers",
+        "macho_otool_expected_diff_libraries",
+        "macho_otool_expected_diff_indirect_symbols",
+        "macho_otool_expected_diff__text",
+        "macho_otool_expected_diff__cstring",
+        "macho_otool_expected_diff__data",
+    ]
+    for idx, diff in enumerate(arch_differences):
+        assert_diff(diff, filenames[idx])
+
+
+@skip_unless_tools_exist("llvm-readobj", "llvm-objdump")
+def test_llvm_obj_compare_non_existing(monkeypatch, obj1):
+    monkeypatch.setattr(Config(), "new_file", True)
+    difference = obj1.compare(MissingFile("/nonexisting", obj1))
+    assert difference.source2 == "/nonexisting"
+    assert len(difference.details) > 0
+
+
+@skip_unless_tools_exist("llvm-readobj", "llvm-objdump")
+def test_llvm_diff(obj_differences):
+    # Headers
+    assert len(obj_differences) == 8
+    filenames = [
+        "macho_llvm_expected_diff_strings",
+        "macho_llvm_expected_diff_file_headers",
+        "macho_llvm_expected_diff_needed_libs",
+        "macho_llvm_expected_diff_symbols",
+        "macho_llvm_expected_diff_dyn_symbols",
+        "macho_llvm_expected_diff_relocations",
+        "macho_llvm_expected_diff_dyn_relocations",
+    ]
+    for idx, diff in enumerate(obj_differences[:-1]):
+        assert_diff(diff, filenames[idx])
+
+    # Sections
+    arch_differences = obj_differences[-1].details
+    assert len(arch_differences) == 7
     filenames = [
-        "macho_expected_diff_arch",
-        "macho_expected_diff_headers",
-        "macho_expected_diff_loadcommands",
-        "macho_expected_diff_disassembly",
+        "macho_llvm_expected_diff__text",
+        "macho_llvm_expected_diff__stubs",
+        "macho_llvm_expected_diff__stub_helper",
+        "macho_llvm_expected_diff__cstring",
+        "macho_llvm_expected_diff__unwind_info",
+        "macho_llvm_expected_diff__eh_frame",
+        "macho_llvm_expected_diff__la_symbol_ptr",
     ]
-    for idx, diff in enumerate(obj_differences):
-        assert diff.unified_diff == get_data(filenames[idx])
+    for idx, diff in enumerate(arch_differences):
+        assert_diff(diff, filenames[idx])

tests/comparators/test_macho_decompiler.py

+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2021 Jean-Romain Garnier <salsa@jean-romain.com>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
+
+import pytest
+
+from diffoscope.config import Config
+from diffoscope.comparators.missing_file import MissingFile
+
+from ..utils.data import load_fixture, assert_diff
+from ..utils.tools import (
+    skipif,
+    tools_missing,
+    skip_unless_tools_exist,
+    skip_unless_module_exists,
+)
+
+
+def radare2_command_is_undefined(x):
+    if tools_missing("radare2"):
+        return True
+
+    try:
+        # Open any file with radare2 and try to execute the given command
+        # If it returns None, then the command doesn't exist
+        import r2pipe
+
+        r2 = r2pipe.open("/dev/null", flags=["-2"])
+        return r2.cmdj(x) is None
+    except ImportError:
+        return True
+
+
+def skip_unless_radare2_command_exists(command):
+    return skipif(
+        radare2_command_is_undefined(command),
+        reason=f"radare2 didn't recognize {command} command",
+        tools=(f"{command}_radare2_command",),
+    )
+
+
+def exclude_commands(monkeypatch, patterns):
+    excluded = list(Config().exclude_commands)
+    excluded += patterns
+    monkeypatch.setattr(Config(), "exclude_commands", excluded)
+
+
+@pytest.fixture(scope="function", autouse=True)
+def init_tests(request, monkeypatch):
+    # Ignore Mach-O tools that are already tested in test_macho.py
+    exclude_commands(
+        monkeypatch,
+        [
+            "^llvm-readobj.*",
+            "^llvm-objdump.*",
+            "^lipo.*",
+            "^otool.*",
+            "^strings.*",
+        ],
+    )
+
+
+obj1 = load_fixture("test1.macho")
+obj2 = load_fixture("test2.macho")
+
+
+@pytest.fixture
+def obj_differences(obj1, obj2):
+    return obj1.compare(obj2).details
+
+
+@skip_unless_tools_exist("radare2")
+@skip_unless_module_exists("r2pipe")
+@skip_unless_radare2_command_exists("pdgj")
+def test_obj_compare_non_existing(monkeypatch, obj1):
+    monkeypatch.setattr(Config(), "new_file", True)
+    difference = obj1.compare(MissingFile("/nonexisting", obj1))
+    assert difference.source2 == "/nonexisting"
+    assert len(difference.details) > 0
+
+
+@skip_unless_tools_exist("radare2")
+@skip_unless_module_exists("r2pipe")
+@skip_unless_radare2_command_exists("pdgj")
+def test_ghidra_diff(monkeypatch, obj1, obj2):
+    exclude_commands(monkeypatch, ["^radare2 disass.*"])
+    obj_differences = obj1.compare(obj2).details[0].details
+    assert len(obj_differences) == 1
+
+    filenames = ["macho_obj_ghidra_expected_diff_main"]
+    for idx, diff in enumerate(obj_differences):
+        assert_diff(diff.details[0], filenames[idx])
+
+
+@skip_unless_tools_exist("radare2")
+@skip_unless_module_exists("r2pipe")
+def test_radare2_diff(monkeypatch, obj1, obj2):
+    exclude_commands(monkeypatch, ["^radare2 r2ghidra.*"])
+    obj_differences = obj1.compare(obj2).details[0].details
+    assert len(obj_differences) == 2
+
+    filenames = [
+        "macho_obj_radare2_expected_diff_main",
+        "macho_obj_radare2_expected_diff_printf",
+    ]
+    for idx, diff in enumerate(obj_differences):
+        assert_diff(diff.details[0], filenames[idx])

tests/data/elf_obj_ghidra_expected_diff

-@@ -1,6 +1,6 @@
+@@ -1,7 +1,7 @@
  
  undefined8 sym.f(void)
+ 
  {
      // [01] -r-x section size 11 named .text
 -    return 0x2a;

tests/data/elf_obj_radare2_expected_diff

-@@ -1,13 +1,13 @@
- function sym.f () {
-     //  1 basic blocks
+@@ -1,11 +1,11 @@
  
+ // This function contains 1 basic blocks and its 11 long.
+ function sym.f () {
      loc_0x8000040:
- 
-        push rbp                 //[01] -r-x section size 11 named .text
-        rbp = rsp
--       eax = 0x2a               //'*' ; 42
-+       eax = 0xffffffff         //-1
-                                 //rsp ; rsp
-        return
- (break)
- 
+     push rbp      //[01] -r-x section size 11 named .text
+     rbp = rsp
+-    eax = 0x2a    //'*' ; 42
++    eax = 0xffffffff //-1
+     //rsp ; rsp
+     return
+     (break)
  }

tests/data/elfmix_mach_o_expected_diff

-@@ -23,15 +23,15 @@
- 00000160: 0100 0000 000e 0a00 000e 0a00 0000 0000  ................
- 00000170: 0200 0000 1800 0000 5002 0000 0100 0000  ........P.......
- 00000180: 6002 0000 1000 0000 0b00 0000 5000 0000  `...........P...
- 00000190: 0000 0000 0000 0000 0000 0000 0100 0000  ................
- 000001a0: 0100 0000 0000 0000 0000 0000 0000 0000  ................
- 000001b0: 0000 0000 0000 0000 0000 0000 0000 0000  ................
- 000001c0: 0000 0000 0000 0000 0000 0000 0000 0000  ................
--000001d0: 0000 0000 0000 0000 5548 89e5 b82a 0000  ........UH...*..
-+000001d0: 0000 0000 0000 0000 5548 89e5 b82b 0000  ........UH...+..
- 000001e0: 005d c300 0000 0000 0000 0000 0000 0000  .]..............
- 000001f0: 0b00 0000 0000 0001 0000 0000 0000 0000  ................
- 00000200: 0000 0000 0000 0000 1400 0000 0000 0000  ................
- 00000210: 017a 5200 0178 1001 100c 0708 9001 0000  .zR..x..........
- 00000220: 2400 0000 1c00 0000 b0ff ffff ffff ffff  $...............
- 00000230: 0b00 0000 0000 0000 0041 0e10 8602 430d  .........A....C.
- 00000240: 0600 0000 0000 0000 0000 0000 0100 0006  ................

tests/data/elfmix_mach_o_expected_diff__text

+@@ -1,7 +1,7 @@
+ Contents of (__TEXT,__text) section
+ _return42_or_3:
+ 	pushq	%rbp
+ 	movq	%rsp, %rbp
+-	movl	$42, %eax
++	movl	$43, %eax
+ 	popq	%rbp
+ 	retq

tests/data/macho_expected_diff_arch

-@@ -1,2 +1 @@
--i386
- x86_64
-

tests/data/macho_llvm_expected_diff__cstring

+@@ -1,4 +1,3 @@
+ Contents of (__TEXT,__cstring) section
+-%s %s\n
+-17:31:50
+-Wed Dec  2 17:31:49 2015
++%s\n
++15:52:34

tests/data/macho_llvm_expected_diff__eh_frame

+@@ -1,3 +1,3 @@
+ Contents of (__TEXT,__eh_frame) section
+-0000000100000fe8	14 00 00 00 00 00 00 00 03 7a 52 00 01 78 10 01 
+-0000000100000ff8	10 0c 07 08 90 01 00 00 
++0000000100000fe0	14 00 00 00 00 00 00 00 03 7a 52 00 01 78 10 01 
++0000000100000ff0	10 0c 07 08 90 01 00 00 

tests/data/macho_llvm_expected_diff__la_symbol_ptr

+@@ -1,3 +1,3 @@
+ Contents of (__DATA,__la_symbol_ptr) section
+ Unknown section type (0x00000007)
+-0000000100001010	68 0f 00 00 01 00 00 00 
++0000000100001010	80 0f 00 00 01 00 00 00 

tests/data/macho_llvm_expected_diff__stub_helper

+@@ -1,7 +1,7 @@
+ Contents of (__TEXT,__stub_helper) section
+-	leaq	169(%rip), %r11
++	leaq	145(%rip), %r11
+ 	pushq	%r11
+-	jmpq	*153(%rip) ## literal pool symbol address: dyld_stub_binder
++	jmpq	*129(%rip) ## literal pool symbol address: dyld_stub_binder
+ 	nop
+ 	pushq	$0
+-	jmp	0x100000f58
++	jmp	0x100000f70

tests/data/macho_llvm_expected_diff__stubs

+@@ -1,2 +1,2 @@
+ Contents of (__TEXT,__stubs) section
+-	jmpq	*184(%rip) ## literal pool symbol address: _printf
++	jmpq	*160(%rip) ## literal pool symbol address: _printf

tests/data/macho_llvm_expected_diff__text

+@@ -1,16 +1,15 @@
+ Contents of (__TEXT,__text) section
+ _main:
+ 	pushq	%rbp
+ 	movq	%rsp, %rbp
+ 	subq	$16, %rsp
+-	leaq	67(%rip), %rdi ## literal pool for: "%s %s\n"
+-	leaq	67(%rip), %rsi ## literal pool for: "17:31:50"
+-	leaq	69(%rip), %rdx ## literal pool for: "Wed Dec  2 17:31:49 2015"
++	leaq	59(%rip), %rdi ## literal pool for: "%s\n"
++	leaq	56(%rip), %rsi ## literal pool for: "15:52:34"
+ 	movb	$0, %al
+-	callq	0x100000f52 ## symbol stub for: _printf
++	callq	0x100000f6a ## symbol stub for: _printf
+ 	xorl	%ecx, %ecx
+ 	movl	%eax, -4(%rbp)
+ 	movl	%ecx, %eax
+ 	addq	$16, %rsp
+ 	popq	%rbp
+ 	retq

tests/data/macho_llvm_expected_diff__unwind_info

+@@ -1,6 +1,6 @@
+ Contents of (__TEXT,__unwind_info) section
+-0000000100000f9c	01 00 00 00 1c 00 00 00 00 00 00 00 1c 00 00 00 
+-0000000100000fac	00 00 00 00 1c 00 00 00 02 00 00 00 20 0f 00 00 
+-0000000100000fbc	34 00 00 00 34 00 00 00 52 0f 00 00 00 00 00 00 
+-0000000100000fcc	34 00 00 00 03 00 00 00 0c 00 01 00 10 00 01 00 
+-0000000100000fdc	00 00 00 00 00 00 00 01 
++0000000100000f98	01 00 00 00 1c 00 00 00 00 00 00 00 1c 00 00 00 
++0000000100000fa8	00 00 00 00 1c 00 00 00 02 00 00 00 40 0f 00 00 
++0000000100000fb8	34 00 00 00 34 00 00 00 6b 0f 00 00 00 00 00 00 
++0000000100000fc8	34 00 00 00 03 00 00 00 0c 00 01 00 10 00 01 00 
++0000000100000fd8	00 00 00 00 00 00 00 01