Skip to content
Commits on Source (3)
......@@ -13,6 +13,7 @@ Build-Depends:
apktool [!ppc64el !s390x] <!nocheck>,
bash-completion,
binutils-multiarch <!nocheck>,
black <!nocheck>,
caca-utils <!nocheck>,
colord <!nocheck>,
db-util <!nocheck>,
......
......@@ -131,10 +131,11 @@ class ComparatorManager(object):
self.classes.append(getattr(mod, klass_name))
break
else: # noqa
logger.error("Could not import {}{}".format(
"any of " if len(xs) > 1 else '',
', '.join(xs)
))
logger.error(
"Could not import {}{}".format(
"any of " if len(xs) > 1 else '', ', '.join(xs)
)
)
for x in errors:
logger.error("Original error for %s:", x[0])
sys.stderr.buffer.write(line_eraser())
......
......@@ -55,8 +55,7 @@ class AndroidBootImgContainer(Archive):
self._unpacked = get_temporary_directory()
logger.debug(
"Extracting Android boot image to %s",
self._unpacked.name,
"Extracting Android boot image to %s", self._unpacked.name
)
subprocess.check_call(
......
......@@ -45,22 +45,34 @@ class ApkContainer(Archive):
def open_archive(self):
self._members = []
self._unpacked = os.path.join(
get_temporary_directory().name,
os.path.basename(self.source.name),
get_temporary_directory().name, os.path.basename(self.source.name)
)
self._andmanifest = None
self._andmanifest_orig = None
logger.debug("Extracting %s to %s", self.source.name, self._unpacked)
subprocess.check_call((
'apktool', 'd', '-k', '-m', '-o', self._unpacked, self.source.path,
), shell=False, stderr=None, stdout=subprocess.PIPE)
subprocess.check_call(
(
'apktool',
'd',
'-k',
'-m',
'-o',
self._unpacked,
self.source.path,
),
shell=False,
stderr=None,
stdout=subprocess.PIPE,
)
# Optionally extract the classes.dex file; apktool does not do this.
subprocess.call((
'unzip', '-d', self._unpacked, self.source.path, 'classes.dex',
), stderr=subprocess.PIPE, stdout=subprocess.PIPE)
subprocess.call(
('unzip', '-d', self._unpacked, self.source.path, 'classes.dex'),
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
for root, _, files in os.walk(self._unpacked):
current_dir = []
......@@ -74,8 +86,7 @@ class ApkContainer(Archive):
# reproducibility.
if filename == 'apktool.yml':
abspath = filter_apk_metadata(
abspath,
os.path.basename(self.source.name),
abspath, os.path.basename(self.source.name)
)
relpath = abspath[len(self._unpacked) + 1 :]
current_dir.insert(0, relpath)
......@@ -98,8 +109,9 @@ class ApkContainer(Archive):
return self
def get_android_manifest(self):
return self.get_member(self._andmanifest) \
if self._andmanifest else None
return (
self.get_member(self._andmanifest) if self._andmanifest else None
)
def get_original_android_manifest(self):
if self._andmanifest_orig:
......@@ -122,21 +134,25 @@ class ApkContainer(Archive):
diff_manifests = None
if my_android_manifest and other_android_manifest:
source = 'AndroidManifest.xml (decoded)'
diff_manifests = compare_files(my_android_manifest,
other_android_manifest,
source=source)
diff_manifests = compare_files(
my_android_manifest, other_android_manifest, source=source
)
if diff_manifests is None:
comment = 'No difference found for decoded AndroidManifest.xml'
else:
comment = 'No decoded AndroidManifest.xml found ' + \
'for one of the APK files.'
comment = (
'No decoded AndroidManifest.xml found '
+ 'for one of the APK files.'
)
if diff_manifests:
return diff_manifests
source = 'AndroidManifest.xml (original / undecoded)'
diff_manifests = compare_files(self.get_original_android_manifest(),
diff_manifests = compare_files(
self.get_original_android_manifest(),
other.get_original_android_manifest(),
source=source)
source=source,
)
if diff_manifests is not None:
diff_manifests.add_comment(comment)
return diff_manifests
......@@ -159,8 +175,9 @@ class ApkFile(File):
CONTAINER_CLASS = ApkContainer
def compare_details(self, other, source=None):
zipinfo_difference = Difference.from_command(Zipinfo, self.path, other.path) or \
Difference.from_command(ZipinfoVerbose, self.path, other.path)
zipinfo_difference = Difference.from_command(
Zipinfo, self.path, other.path
) or Difference.from_command(ZipinfoVerbose, self.path, other.path)
return [zipinfo_difference]
......@@ -170,7 +187,7 @@ def filter_apk_metadata(filepath, archive_name):
logger.debug("Moving APK metadata from %s to %s", filepath, new_filename)
re_filename = re.compile(
r'^apkFileName: %s' % re.escape(os.path.basename(archive_name)),
r'^apkFileName: %s' % re.escape(os.path.basename(archive_name))
)
with open(filepath) as in_, open(new_filename, 'w') as out:
......
......@@ -36,6 +36,7 @@ logger = logging.getLogger(__name__)
# ArFile gives slightly more reasonable output, e.g. a readable plain diff of
# the __.PKGDEF member which is just a text file containing the Go interface.
class ArContainer(LibarchiveContainer):
def get_adjusted_members(self):
members = list(super().get_adjusted_members())
......@@ -46,8 +47,9 @@ class ArContainer(LibarchiveContainer):
filtered_out = [p for p in members if p[0] in known_ignores]
if filtered_out:
for k, v in filtered_out:
logger.debug("ignored ar member '%s' because %s",
k, known_ignores[k])
logger.debug(
"ignored ar member '%s' because %s", k, known_ignores[k]
)
return [p for p in members if p[0] not in known_ignores]
......@@ -63,7 +65,15 @@ class ArFile(File):
FILE_TYPE_RE = re.compile(r'\bar archive\b')
def compare_details(self, other, source=None):
return [Difference.from_command(ArSymbolTableDumper, self.path, other.path),
Difference.from_text_readers(list_libarchive(self.path),
return [
Difference.from_command(
ArSymbolTableDumper, self.path, other.path
),
Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path, other.path, source="file list")]
self.path,
other.path,
source="file list",
),
]
......@@ -46,9 +46,8 @@ class BerkeleyDBFile(File):
FILE_TYPE_RE = re.compile(r'^Berkeley DB ')
def compare_details(self, other, source=None):
return [Difference.from_command(
DbDump,
self.path,
other.path,
source="Berkeley DB file",
)]
return [
Difference.from_command(
DbDump, self.path, other.path, source="Berkeley DB file"
)
]
......@@ -43,6 +43,7 @@ else:
if not hasattr(fn, '_temp_dir'):
fn._temp_dir = get_temporary_directory('binwalk').name
return fn._temp_dir
binwalk.core.settings.Settings._get_user_config_dir = fn
logger = logging.getLogger(__name__)
......
......@@ -46,7 +46,10 @@ class Bzip2Container(Archive):
with open(dest_path, 'wb') as fp:
subprocess.check_call(
["bzip2", "--decompress", "--stdout", self.source.path],
shell=False, stdout=fp, stderr=subprocess.PIPE)
shell=False,
stdout=fp,
stderr=subprocess.PIPE,
)
return dest_path
......
......@@ -38,7 +38,8 @@ class CbfsListing(Command):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._header_re = re.compile(
r'^.*: ([^,]+, bootblocksize [0-9]+, romsize [0-9]+, offset 0x[0-9A-Fa-f]+)$')
r'^.*: ([^,]+, bootblocksize [0-9]+, romsize [0-9]+, offset 0x[0-9A-Fa-f]+)$'
)
@tool_required('cbfstool')
def cmdline(self):
......@@ -76,11 +77,19 @@ class CbfsContainer(Archive):
@tool_required('cbfstool')
def extract(self, member_name, dest_dir):
dest_path = os.path.join(dest_dir, os.path.basename(member_name))
cmd = ['cbfstool', self.source.path, 'extract',
'-n', member_name, '-f', dest_path]
cmd = [
'cbfstool',
self.source.path,
'extract',
'-n',
member_name,
'-f',
dest_path,
]
logger.debug("cbfstool extract %s to %s", member_name, dest_path)
subprocess.check_call(
cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
)
return dest_path
......@@ -95,11 +104,16 @@ CBFS_MAXIMUM_FILE_SIZE = 24 * 2 ** 20 # 24 MiB
def is_header_valid(buf, size, offset=0):
magic, version, romsize, bootblocksize, align, cbfs_offset, architecture, pad = struct.unpack_from(
'!IIIIIIII', buf, offset)
return magic == CBFS_HEADER_MAGIC and \
(version == CBFS_HEADER_VERSION1 or version == CBFS_HEADER_VERSION2) and \
(romsize <= size) and \
(cbfs_offset < romsize)
'!IIIIIIII', buf, offset
)
return (
magic == CBFS_HEADER_MAGIC
and (
version == CBFS_HEADER_VERSION1 or version == CBFS_HEADER_VERSION2
)
and (romsize <= size)
and (cbfs_offset < romsize)
)
class CbfsFile(File):
......@@ -118,7 +132,11 @@ class CbfsFile(File):
# that field is now bound to be little endian
# -- #coreboot, 2015-10-14
rel_offset = struct.unpack('<i', f.read(4))[0]
if rel_offset < 0 and -rel_offset > CBFS_HEADER_SIZE and -rel_offset < size:
if (
rel_offset < 0
and -rel_offset > CBFS_HEADER_SIZE
and -rel_offset < size
):
f.seek(rel_offset, io.SEEK_END)
logger.debug('looking for header at offset: %x', f.tell())
if is_header_valid(f.read(CBFS_HEADER_SIZE), size):
......@@ -127,7 +145,8 @@ class CbfsFile(File):
return False
else:
logger.debug(
'CBFS relative offset seems wrong, scanning whole image')
'CBFS relative offset seems wrong, scanning whole image'
)
f.seek(0, io.SEEK_SET)
offset = 0
buf = f.read(CBFS_HEADER_SIZE)
......
......@@ -32,10 +32,12 @@ class CpioFile(File):
FILE_TYPE_RE = re.compile(r'\bcpio archive\b')
def compare_details(self, other, source=None):
return [Difference.from_text_readers(
return [
Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path,
other.path,
source="file list",
)]
)
]
......@@ -98,8 +98,11 @@ class DebContainer(LibarchiveContainer):
for name1 in my_members.keys():
main, ext = os.path.splitext(name1)
candidates = [name2 for name2 in other_members.keys() - matched
if os.path.splitext(name2)[0] == main]
candidates = [
name2
for name2 in other_members.keys() - matched
if os.path.splitext(name2)[0] == main
]
if len(candidates) == 1:
yield name1, candidates[0], 0
matched.add(candidates[0])
......@@ -113,8 +116,11 @@ class DebFile(File):
def md5sums(self):
if not hasattr(self, '_md5sums'):
control_tar = self.as_container.control_tar
md5sums_file = control_tar.as_container.lookup_file(
'./md5sums') if control_tar else None
md5sums_file = (
control_tar.as_container.lookup_file('./md5sums')
if control_tar
else None
)
if isinstance(md5sums_file, Md5sumsFile):
self._md5sums = md5sums_file.parse()
else:
......@@ -129,7 +135,8 @@ class DebFile(File):
if not hasattr(self, '_control'):
control_file = self.as_container.control_tar.as_container.lookup_file(
'./control')
'./control'
)
if control_file:
with open(control_file.path, 'rb') as f:
self._control = deb822.Deb822(f)
......@@ -137,21 +144,35 @@ class DebFile(File):
return self._control
def compare_details(self, other, source=None):
return [Difference.from_text_readers(list_libarchive(self.path),
return [
Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path, other.path, source="file list")]
self.path,
other.path,
source="file list",
)
]
class Md5sumsFile(File):
@classmethod
def recognizes(cls, file):
return isinstance(file, ArchiveMember) and \
file.name == './md5sums' and \
isinstance(file.container.source, ArchiveMember) and \
isinstance(file.container.source.container.source, ArchiveMember) and \
DebContainer.RE_CONTROL_TAR.match(file.container.source.container.source.name) and \
isinstance(
file.container.source.container.source.container.source, DebFile)
return (
isinstance(file, ArchiveMember)
and file.name == './md5sums'
and isinstance(file.container.source, ArchiveMember)
and isinstance(
file.container.source.container.source, ArchiveMember
)
and DebContainer.RE_CONTROL_TAR.match(
file.container.source.container.source.name
)
and isinstance(
file.container.source.container.source.container.source,
DebFile,
)
)
def parse(self):
try:
......@@ -171,9 +192,18 @@ class Md5sumsFile(File):
yield " ".join(line.split(" ")[2:])
def compare_details(self, other, source=None):
return [compare_files(self, other, source='md5sums', diff_content_only=True),
Difference.from_text_readers(self.strip_checksum(self.path), self.strip_checksum(other.path),
self.path, other.path, source="line order")]
return [
compare_files(
self, other, source='md5sums', diff_content_only=True
),
Difference.from_text_readers(
self.strip_checksum(self.path),
self.strip_checksum(other.path),
self.path,
other.path,
source="line order",
),
]
class DebTarContainer(TarContainer):
......@@ -184,12 +214,17 @@ class DebTarContainer(TarContainer):
if self.source:
my_md5sums = self.source.container.source.container.source.md5sums
if other.source:
other_md5sums = other.source.container.source.container.source.md5sums
other_md5sums = (
other.source.container.source.container.source.md5sums
)
for my_member, other_member, comment in super().comparisons(other):
if not Config().force_details and \
my_member.name == other_member.name and \
my_md5sums.get(my_member.name, 'my') == other_md5sums.get(other_member.name, 'other'):
if (
not Config().force_details
and my_member.name == other_member.name
and my_md5sums.get(my_member.name, 'my')
== other_md5sums.get(other_member.name, 'other')
):
logger.debug("Skip %s: identical md5sum", my_member.name)
continue
yield my_member, other_member, comment
......@@ -200,12 +235,20 @@ class DebDataTarFile(File):
@classmethod
def recognizes(cls, file):
return isinstance(file, ArchiveMember) and \
isinstance(file.container.source, ArchiveMember) and \
DebContainer.RE_DATA_TAR.match(file.container.source.name) and \
isinstance(file.container.source.container.source, DebFile)
return (
isinstance(file, ArchiveMember)
and isinstance(file.container.source, ArchiveMember)
and DebContainer.RE_DATA_TAR.match(file.container.source.name)
and isinstance(file.container.source.container.source, DebFile)
)
def compare_details(self, other, source=None):
return [Difference.from_text_readers(list_libarchive(self.path, ignore_errors=True),
return [
Difference.from_text_readers(
list_libarchive(self.path, ignore_errors=True),
list_libarchive(other.path, ignore_errors=True),
self.path, other.path, source="file list")]
self.path,
other.path,
source="file list",
)
]
......@@ -51,8 +51,7 @@ class DebControlMember(File):
@property
def path(self):
return os.path.join(
os.path.dirname(self.container.source.path),
self.name,
os.path.dirname(self.container.source.path), self.name
)
def is_directory(self):
......@@ -79,10 +78,9 @@ class DebControlContainer(Container):
if '-' in version:
upstream, revision = version.rsplit('-', 1)
return re.compile(r'_%s(?:-%s)?' % (
re.escape(upstream),
re.escape(revision),
))
return re.compile(
r'_%s(?:-%s)?' % (re.escape(upstream), re.escape(revision))
)
return re.compile(re.escape(version))
......@@ -91,8 +89,9 @@ class DebControlContainer(Container):
yield self._trim_version_number(name), self.get_member(name)
def get_member_names(self):
field = self.source.deb822.get('Files') or \
self.source.deb822.get('Checksums-Sha256')
field = self.source.deb822.get('Files') or self.source.deb822.get(
'Checksums-Sha256'
)
# Show results from debugging packages last; they are rather verbose,
# masking other more interesting differences due to truncating the
......@@ -119,7 +118,9 @@ class DebControlFile(File):
def compare_details(self, other, source=None):
differences = []
for field in sorted(set(self.deb822.keys()).union(set(other.deb822.keys()))):
for field in sorted(
set(self.deb822.keys()).union(set(other.deb822.keys()))
):
if field.startswith('Checksums-') or field == 'Files':
continue
......@@ -131,31 +132,33 @@ class DebControlFile(File):
if field in other.deb822:
other_value = other.deb822.get_as_string(field).lstrip()
differences.append(Difference.from_text(
my_value,
other_value,
self.path,
other.path,
source=field,
))
differences.append(
Difference.from_text(
my_value, other_value, self.path, other.path, source=field
)
)
# Compare Files as string
if self.deb822.get('Files'):
differences.append(Difference.from_text(
differences.append(
Difference.from_text(
self.deb822.get_as_string('Files'),
other.deb822.get_as_string('Files'),
self.path,
other.path,
source='Files',
))
)
)
else:
differences.append(Difference.from_text(
differences.append(
Difference.from_text(
self.deb822.get_as_string('Checksums-Sha256'),
other.deb822.get_as_string('Checksums-Sha256'),
self.path,
other.path,
source='Checksums-Sha256',
))
)
)
return differences
......@@ -189,13 +192,14 @@ class DotChangesFile(DebControlFile):
files = zip(self.deb822.get('Files'), other.deb822.get('Files'))
files_identical = all(
x == y for x, y in files
if not x['name'].endswith('.buildinfo')
x == y for x, y in files if not x['name'].endswith('.buildinfo')
)
if files_identical and \
len(differences.details) == 1 and \
differences.details[0].source1 == 'Files':
if (
files_identical
and len(differences.details) == 1
and differences.details[0].source1 == 'Files'
):
logger.warning("Ignoring buildinfo file differences")
return None
......@@ -219,8 +223,7 @@ class DotDscFile(DebControlFile):
# XXX: this will not work for containers
in_dsc_path = os.path.join(
os.path.dirname(file.path),
d['Name'],
os.path.dirname(file.path), d['Name']
)
if not os.path.exists(in_dsc_path):
return False
......@@ -277,8 +280,7 @@ class DotBuildinfoFile(DebControlFile):
# XXX: this will not work for containers
in_buildinfo_path = os.path.join(
os.path.dirname(file.path),
d['Name'],
os.path.dirname(file.path), d['Name']
)
if not os.path.exists(in_buildinfo_path):
return False
......
......@@ -28,7 +28,8 @@ class DotChangesFile(TextFile):
if not difference:
return None
difference.add_comment(
'Unable to find Python debian module. Falling back to text comparison.')
'Unable to find Python debian module. Falling back to text comparison.'
)
return difference
......@@ -40,7 +41,8 @@ class DotDscFile(TextFile):
if not difference:
return None
difference.add_comment(
'Unable to find Python debian module. Falling back to text comparison.')
'Unable to find Python debian module. Falling back to text comparison.'
)
return difference
......@@ -52,5 +54,6 @@ class DotBuildinfoFile(TextFile):
if not difference:
return None
difference.add_comment(
'Unable to find Python debian module. Falling back to text comparison.')
'Unable to find Python debian module. Falling back to text comparison.'
)
return difference
......@@ -70,9 +70,15 @@ class Device(File):
super().cleanup()
def compare(self, other, source=None):
with open(self.path) as my_content, \
open(other.path) as other_content:
return Difference.from_text_readers(my_content, other_content, self.name, other.name, source=source, comment="device")
with open(self.path) as my_content, open(other.path) as other_content:
return Difference.from_text_readers(
my_content,
other_content,
self.name,
other.name,
source=source,
comment="device",
)
def format_device(mode, major, minor):
......
......@@ -48,8 +48,12 @@ class DexContainer(Archive):
def extract(self, member_name, dest_dir):
dest_path = os.path.join(dest_dir, member_name)
logger.debug('dex extracting to %s', dest_path)
subprocess.check_call(['enjarify', '-o', dest_path, self.source.path],
shell=False, stderr=None, stdout=subprocess.PIPE)
subprocess.check_call(
['enjarify', '-o', dest_path, self.source.path],
shell=False,
stderr=None,
stdout=subprocess.PIPE,
)
return dest_path
......
......@@ -41,25 +41,33 @@ def list_files(path):
path = os.path.realpath(path)
all_files = []
for root, dirs, names in os.walk(path):
all_files.extend([os.path.join(root[len(path) + 1:], dir)
for dir in dirs])
all_files.extend([os.path.join(root[len(path) + 1:], name)
for name in names])
all_files.extend(
[os.path.join(root[len(path) + 1 :], dir) for dir in dirs]
)
all_files.extend(
[os.path.join(root[len(path) + 1 :], name) for name in names]
)
all_files.sort()
return all_files
if os.uname()[0] == 'FreeBSD':
class Stat(Command):
@tool_required('stat')
def cmdline(self):
return [
'stat',
'-t', '%Y-%m-%d %H:%M:%S',
'-f', '%Sp %l %Su %Sg %z %Sm %k %b %#Xf',
'-t',
'%Y-%m-%d %H:%M:%S',
'-f',
'%Sp %l %Su %Sg %z %Sm %k %b %#Xf',
self.path,
]
else:
class Stat(Command):
@tool_required('stat')
def cmdline(self):
......@@ -90,9 +98,7 @@ def lsattr(path):
try:
output = subprocess.check_output(
['lsattr', '-d', path],
shell=False,
stderr=subprocess.STDOUT,
['lsattr', '-d', path], shell=False, stderr=subprocess.STDOUT
).decode('utf-8')
return output.split()[0]
except subprocess.CalledProcessError as e:
......@@ -122,24 +128,28 @@ def xattr(path1, path2):
try:
get_all = xattr_.get_all
except AttributeError:
def get_all(x):
return xattr_.xattr(x).items()
def fn(x):
return '\n'.join('{}: {}'.format(
k.decode('utf-8', 'ignore'),
v.decode('utf-8', 'ignore'),
) for k, v in get_all(x))
return '\n'.join(
'{}: {}'.format(
k.decode('utf-8', 'ignore'), v.decode('utf-8', 'ignore')
)
for k, v in get_all(x)
)
return Difference.from_text(
fn(path1), fn(path2), path1, path2, source='extended file attributes',
fn(path1), fn(path2), path1, path2, source='extended file attributes'
)
def compare_meta(path1, path2):
if Config().exclude_directory_metadata in ('yes', 'recursive'):
logger.debug(
"Excluding directory metadata for paths (%s, %s)", path1, path2)
"Excluding directory metadata for paths (%s, %s)", path1, path2
)
return []
logger.debug('compare_meta(%s, %s)', path1, path2)
......@@ -159,20 +169,20 @@ def compare_meta(path1, path2):
differences.append(Difference.from_command(Getfacl, path1, path2))
except RequiredToolNotFound:
logger.info(
"Unable to find 'getfacl', some directory metadata differences might not be noticed.")
"Unable to find 'getfacl', some directory metadata differences might not be noticed."
)
try:
lsattr1 = lsattr(path1)
lsattr2 = lsattr(path2)
differences.append(Difference.from_text(
lsattr1,
lsattr2,
path1,
path2,
source='lsattr',
))
differences.append(
Difference.from_text(
lsattr1, lsattr2, path1, path2, source='lsattr'
)
)
except RequiredToolNotFound:
logger.info(
"Unable to find 'lsattr', some directory metadata differences might not be noticed.")
"Unable to find 'lsattr', some directory metadata differences might not be noticed."
)
differences.append(xattr(path1, path2))
return [d for d in differences if d is not None]
......@@ -256,16 +266,17 @@ class DirectoryContainer(Container):
return FilesystemDirectory(member_path)
return FilesystemFile(
os.path.join(self.source.path, member_name),
container=self,
os.path.join(self.source.path, member_name), container=self
)
def comparisons(self, other):
my_members = collections.OrderedDict(self.get_adjusted_members_sizes())
other_members = collections.OrderedDict(
other.get_adjusted_members_sizes())
total_size = sum(x[1] for x in my_members.values()) + \
sum(x[1] for x in other_members.values())
other.get_adjusted_members_sizes()
)
total_size = sum(x[1] for x in my_members.values()) + sum(
x[1] for x in other_members.values()
)
to_compare = set(my_members.keys()).intersection(other_members.keys())
with Progress(total_size) as p:
......@@ -288,6 +299,5 @@ class DirectoryContainer(Container):
return inner_difference
return filter(
None,
itertools.starmap(compare_pair, self.comparisons(other)),
None, itertools.starmap(compare_pair, self.comparisons(other))
)
......@@ -29,11 +29,7 @@ from .utils.command import Command
class Docx2txt(Command):
@tool_required('docx2txt')
def cmdline(self):
return (
'docx2txt',
self.path,
'-',
)
return ('docx2txt', self.path, '-')
class DocxFile(File):
......@@ -41,9 +37,8 @@ class DocxFile(File):
FILE_TYPE_RE = re.compile(r'^Microsoft Word 2007+\b')
def compare_details(self, other, source=None):
return [Difference.from_command(
Docx2txt,
self.path,
other.path,
source='docx2txt',
)]
return [
Difference.from_command(
Docx2txt, self.path, other.path, source='docx2txt'
)
]
......@@ -38,4 +38,6 @@ class DeviceTreeFile(File):
FILE_TYPE_RE = re.compile(r'^Device Tree Blob')
def compare_details(self, other, source=None):
return [Difference.from_command(DeviceTreeContents, self.path, other.path)]
return [
Difference.from_command(DeviceTreeContents, self.path, other.path)
]
......@@ -63,7 +63,11 @@ class Readelf(Command):
@tool_required('readelf')
def cmdline(self):
return [get_tool_name('readelf'), '--wide'] + self.readelf_options() + [self.path]
return (
[get_tool_name('readelf'), '--wide']
+ self.readelf_options()
+ [self.path]
)
def readelf_options(self):
return [] # noqa
......@@ -190,12 +194,16 @@ class ReadElfSection(Readelf):
return self._section_name
def readelf_options(self):
return ReadElfSection.base_options() + ['--hex-dump={}'.format(self.section_name)]
return ReadElfSection.base_options() + [
'--hex-dump={}'.format(self.section_name)
]
class ReadelfStringSection(ReadElfSection):
def readelf_options(self):
return ReadElfSection.base_options() + ['--string-dump={}'.format(self.section_name)]
return ReadElfSection.base_options() + [
'--string-dump={}'.format(self.section_name)
]
class ObjdumpSection(Command):
......@@ -210,12 +218,11 @@ class ObjdumpSection(Command):
@tool_required('objdump')
def cmdline(self):
return [
get_tool_name('objdump'),
] + self.objdump_options() + [
'--section={}'.format(self._section_name),
self.path,
]
return (
[get_tool_name('objdump')]
+ self.objdump_options()
+ ['--section={}'.format(self._section_name), self.path]
)
def filter(self, line):
# Remove the filename from the output
......@@ -228,7 +235,9 @@ class ObjdumpSection(Command):
class ObjdumpDisassembleSection(ObjdumpSection):
RE_SYMBOL_COMMENT = re.compile(rb'^( +[0-9a-f]+:[^#]+)# [0-9a-f]+ <[^>]+>$')
RE_SYMBOL_COMMENT = re.compile(
rb'^( +[0-9a-f]+:[^#]+)# [0-9a-f]+ <[^>]+>$'
)
def objdump_options(self):
# With '--line-numbers' we get the source filename and line within the
......@@ -290,8 +299,7 @@ class ElfSection(File):
@property
def progress_name(self):
return "{} [{}]".format(
self.container.source.progress_name,
super().progress_name,
self.container.source.progress_name, super().progress_name
)
@property
......@@ -325,10 +333,7 @@ class ElfSection(File):
def compare(self, other, source=None):
return Difference.from_command(
ReadElfSection,
self.path,
other.path,
command_args=[self._name],
ReadElfSection, self.path, other.path, command_args=[self._name]
)
......@@ -377,8 +382,11 @@ def get_build_id(path):
logger.debug("Unable to get Build ID for %s: %s", path, e)
return None
m = re.search(r'^\s+Build ID: ([0-9a-f]+)$',
output.decode('utf-8'), flags=re.MULTILINE)
m = re.search(
r'^\s+Build ID: ([0-9a-f]+)$',
output.decode('utf-8'),
flags=re.MULTILINE,
)
if not m:
return None
......@@ -396,8 +404,11 @@ def get_debug_link(path):
logger.debug("Unable to get Build Id for %s: %s", path, e)
return None
m = re.search(r'^\s+\[\s+0\]\s+(\S+)$', output.decode('utf-8',
errors='replace'), flags=re.MULTILINE)
m = re.search(
r'^\s+\[\s+0\]\s+(\S+)$',
output.decode('utf-8', errors='replace'),
flags=re.MULTILINE,
)
if not m:
return None
......@@ -425,7 +436,8 @@ class ElfContainer(Container):
self.source.path,
]
output = subprocess.check_output(
cmd, shell=False, stderr=subprocess.DEVNULL)
cmd, shell=False, stderr=subprocess.DEVNULL
)
has_debug_symbols = False
try:
......@@ -454,18 +466,23 @@ class ElfContainer(Container):
# Use first match, with last option being '_' as fallback
elf_class = [
ElfContainer.SECTION_FLAG_MAPPING[x]
for x in flags if x in ElfContainer.SECTION_FLAG_MAPPING
for x in flags
if x in ElfContainer.SECTION_FLAG_MAPPING
][0]
logger.debug("Adding section %s (%s) as %s",
name, type, elf_class)
logger.debug(
"Adding section %s (%s) as %s", name, type, elf_class
)
self._sections[name] = elf_class(self, name)
except Exception as e:
command = ' '.join(cmd)
logger.debug(
"OutputParsingError in %s from `%s` output - %s:%s",
self.__class__.__name__, command, e.__class__.__name__, e,
self.__class__.__name__,
command,
e.__class__.__name__,
e,
)
raise OutputParsingError(command, self)
......@@ -476,7 +493,9 @@ class ElfContainer(Container):
def _install_debug_symbols(self):
# Figure out if we are in a Debian package first
try:
deb = self.source.container.source.container.source.container.source
deb = (
self.source.container.source.container.source.container.source
)
except AttributeError:
return
......@@ -490,8 +509,10 @@ class ElfContainer(Container):
# .changes or .buildinfo file). In this case, don't automatically
# search for a -dbgsym file unless the user specified
# `Config().use_dbgsym`.
if not hasattr(deb.container.source, 'container') and \
not Config().use_dbgsym:
if (
not hasattr(deb.container.source, 'container')
and not Config().use_dbgsym
):
return
# Retrieve the Build ID for the ELF file we are examining
......@@ -512,19 +533,24 @@ class ElfContainer(Container):
if build_id not in deb.container.dbgsym_build_id_map:
logger.debug(
'Unable to find a matching debug package for Build Id %s', build_id)
'Unable to find a matching debug package for Build Id %s',
build_id,
)
return
dbgsym_package = deb.container.dbgsym_build_id_map[build_id]
debug_file_path = './usr/lib/debug/.build-id/{0}/{1}.debug'.format(
build_id[:2],
build_id[2:],
build_id[:2], build_id[2:]
)
debug_file = dbgsym_package.as_container.data_tar.as_container.lookup_file(
debug_file_path)
debug_file_path
)
if not debug_file:
logger.debug('Unable to find the matching debug file %s in %s',
debug_file_path, dbgsym_package)
logger.debug(
'Unable to find the matching debug file %s in %s',
debug_file_path,
dbgsym_package,
)
return
# Create a .debug directory and link the debug symbols there with the
......@@ -548,11 +574,12 @@ class ElfContainer(Container):
# 1. Use objcopy to create a file with only the original .gnu_debuglink
# section as we will have to update it to get the CRC right.
debuglink_path = get_named_temporary_file(
prefix='{}.debuglink.'.format(self.source.path),
prefix='{}.debuglink.'.format(self.source.path)
).name
objcopy('--only-section=.gnu_debuglink',
self.source.path, debuglink_path)
objcopy(
'--only-section=.gnu_debuglink', self.source.path, debuglink_path
)
# 2. Monkey-patch the ElfSection object created for the .gnu_debuglink
# to change the path to point to this new file
......@@ -562,6 +589,7 @@ class ElfContainer(Container):
@property
def path(self):
return debuglink_path
section.__class__ = MonkeyPatchedElfSection
# 3. Create a file with the debug symbols in uncompressed form
......@@ -598,12 +626,14 @@ class StaticLibFile(File):
ENABLE_FALLBACK_RECOGONIZES = False
def compare_details(self, other, source=None):
differences = [Difference.from_text_readers(
differences = [
Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path,
other.path,
source="file list",
)]
)
]
differences.extend(_compare_elf_data(self.path, other.path))
return differences
......@@ -58,9 +58,8 @@ class FfprobeFile(File):
FILE_TYPE_RE = re.compile(r'^Audio file')
def compare_details(self, other, source=None):
return [Difference.from_command(
Ffprobe,
self.path,
other.path,
source='ffprobe',
)]
return [
Difference.from_command(
Ffprobe, self.path, other.path, source='ffprobe'
)
]