Commit 21d75462 authored by Chris Lamb's avatar Chris Lamb 💬

Run the black code formatter against the source.

parent 60508932
......@@ -131,10 +131,11 @@ class ComparatorManager(object):
self.classes.append(getattr(mod, klass_name))
break
else: # noqa
logger.error("Could not import {}{}".format(
"any of " if len(xs) > 1 else '',
', '.join(xs)
))
logger.error(
"Could not import {}{}".format(
"any of " if len(xs) > 1 else '', ', '.join(xs)
)
)
for x in errors:
logger.error("Original error for %s:", x[0])
sys.stderr.buffer.write(line_eraser())
......
......@@ -55,8 +55,7 @@ class AndroidBootImgContainer(Archive):
self._unpacked = get_temporary_directory()
logger.debug(
"Extracting Android boot image to %s",
self._unpacked.name,
"Extracting Android boot image to %s", self._unpacked.name
)
subprocess.check_call(
......
......@@ -45,22 +45,34 @@ class ApkContainer(Archive):
def open_archive(self):
self._members = []
self._unpacked = os.path.join(
get_temporary_directory().name,
os.path.basename(self.source.name),
get_temporary_directory().name, os.path.basename(self.source.name)
)
self._andmanifest = None
self._andmanifest_orig = None
logger.debug("Extracting %s to %s", self.source.name, self._unpacked)
subprocess.check_call((
'apktool', 'd', '-k', '-m', '-o', self._unpacked, self.source.path,
), shell=False, stderr=None, stdout=subprocess.PIPE)
subprocess.check_call(
(
'apktool',
'd',
'-k',
'-m',
'-o',
self._unpacked,
self.source.path,
),
shell=False,
stderr=None,
stdout=subprocess.PIPE,
)
# Optionally extract the classes.dex file; apktool does not do this.
subprocess.call((
'unzip', '-d', self._unpacked, self.source.path, 'classes.dex',
), stderr=subprocess.PIPE, stdout=subprocess.PIPE)
subprocess.call(
('unzip', '-d', self._unpacked, self.source.path, 'classes.dex'),
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
for root, _, files in os.walk(self._unpacked):
current_dir = []
......@@ -74,17 +86,16 @@ class ApkContainer(Archive):
# reproducibility.
if filename == 'apktool.yml':
abspath = filter_apk_metadata(
abspath,
os.path.basename(self.source.name),
abspath, os.path.basename(self.source.name)
)
relpath = abspath[len(self._unpacked) + 1:]
relpath = abspath[len(self._unpacked) + 1 :]
current_dir.insert(0, relpath)
continue
relpath = abspath[len(self._unpacked) + 1:]
relpath = abspath[len(self._unpacked) + 1 :]
if filename == 'AndroidManifest.xml':
containing_dir = root[len(self._unpacked) + 1:]
containing_dir = root[len(self._unpacked) + 1 :]
if containing_dir == 'original':
self._andmanifest_orig = relpath
if containing_dir == '':
......@@ -98,8 +109,9 @@ class ApkContainer(Archive):
return self
def get_android_manifest(self):
return self.get_member(self._andmanifest) \
if self._andmanifest else None
return (
self.get_member(self._andmanifest) if self._andmanifest else None
)
def get_original_android_manifest(self):
if self._andmanifest_orig:
......@@ -122,21 +134,25 @@ class ApkContainer(Archive):
diff_manifests = None
if my_android_manifest and other_android_manifest:
source = 'AndroidManifest.xml (decoded)'
diff_manifests = compare_files(my_android_manifest,
other_android_manifest,
source=source)
diff_manifests = compare_files(
my_android_manifest, other_android_manifest, source=source
)
if diff_manifests is None:
comment = 'No difference found for decoded AndroidManifest.xml'
else:
comment = 'No decoded AndroidManifest.xml found ' + \
'for one of the APK files.'
comment = (
'No decoded AndroidManifest.xml found '
+ 'for one of the APK files.'
)
if diff_manifests:
return diff_manifests
source = 'AndroidManifest.xml (original / undecoded)'
diff_manifests = compare_files(self.get_original_android_manifest(),
other.get_original_android_manifest(),
source=source)
diff_manifests = compare_files(
self.get_original_android_manifest(),
other.get_original_android_manifest(),
source=source,
)
if diff_manifests is not None:
diff_manifests.add_comment(comment)
return diff_manifests
......@@ -159,8 +175,9 @@ class ApkFile(File):
CONTAINER_CLASS = ApkContainer
def compare_details(self, other, source=None):
zipinfo_difference = Difference.from_command(Zipinfo, self.path, other.path) or \
Difference.from_command(ZipinfoVerbose, self.path, other.path)
zipinfo_difference = Difference.from_command(
Zipinfo, self.path, other.path
) or Difference.from_command(ZipinfoVerbose, self.path, other.path)
return [zipinfo_difference]
......@@ -170,7 +187,7 @@ def filter_apk_metadata(filepath, archive_name):
logger.debug("Moving APK metadata from %s to %s", filepath, new_filename)
re_filename = re.compile(
r'^apkFileName: %s' % re.escape(os.path.basename(archive_name)),
r'^apkFileName: %s' % re.escape(os.path.basename(archive_name))
)
with open(filepath) as in_, open(new_filename, 'w') as out:
......
......@@ -36,6 +36,7 @@ logger = logging.getLogger(__name__)
# ArFile gives slightly more reasonable output, e.g. a readable plain diff of
# the __.PKGDEF member which is just a text file containing the Go interface.
class ArContainer(LibarchiveContainer):
def get_adjusted_members(self):
members = list(super().get_adjusted_members())
......@@ -46,8 +47,9 @@ class ArContainer(LibarchiveContainer):
filtered_out = [p for p in members if p[0] in known_ignores]
if filtered_out:
for k, v in filtered_out:
logger.debug("ignored ar member '%s' because %s",
k, known_ignores[k])
logger.debug(
"ignored ar member '%s' because %s", k, known_ignores[k]
)
return [p for p in members if p[0] not in known_ignores]
......@@ -63,7 +65,15 @@ class ArFile(File):
FILE_TYPE_RE = re.compile(r'\bar archive\b')
def compare_details(self, other, source=None):
return [Difference.from_command(ArSymbolTableDumper, self.path, other.path),
Difference.from_text_readers(list_libarchive(self.path),
list_libarchive(other.path),
self.path, other.path, source="file list")]
return [
Difference.from_command(
ArSymbolTableDumper, self.path, other.path
),
Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path,
other.path,
source="file list",
),
]
......@@ -46,9 +46,8 @@ class BerkeleyDBFile(File):
FILE_TYPE_RE = re.compile(r'^Berkeley DB ')
def compare_details(self, other, source=None):
return [Difference.from_command(
DbDump,
self.path,
other.path,
source="Berkeley DB file",
)]
return [
Difference.from_command(
DbDump, self.path, other.path, source="Berkeley DB file"
)
]
......@@ -43,6 +43,7 @@ else:
if not hasattr(fn, '_temp_dir'):
fn._temp_dir = get_temporary_directory('binwalk').name
return fn._temp_dir
binwalk.core.settings.Settings._get_user_config_dir = fn
logger = logging.getLogger(__name__)
......
......@@ -46,7 +46,10 @@ class Bzip2Container(Archive):
with open(dest_path, 'wb') as fp:
subprocess.check_call(
["bzip2", "--decompress", "--stdout", self.source.path],
shell=False, stdout=fp, stderr=subprocess.PIPE)
shell=False,
stdout=fp,
stderr=subprocess.PIPE,
)
return dest_path
......
......@@ -38,7 +38,8 @@ class CbfsListing(Command):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._header_re = re.compile(
r'^.*: ([^,]+, bootblocksize [0-9]+, romsize [0-9]+, offset 0x[0-9A-Fa-f]+)$')
r'^.*: ([^,]+, bootblocksize [0-9]+, romsize [0-9]+, offset 0x[0-9A-Fa-f]+)$'
)
@tool_required('cbfstool')
def cmdline(self):
......@@ -76,11 +77,19 @@ class CbfsContainer(Archive):
@tool_required('cbfstool')
def extract(self, member_name, dest_dir):
dest_path = os.path.join(dest_dir, os.path.basename(member_name))
cmd = ['cbfstool', self.source.path, 'extract',
'-n', member_name, '-f', dest_path]
cmd = [
'cbfstool',
self.source.path,
'extract',
'-n',
member_name,
'-f',
dest_path,
]
logger.debug("cbfstool extract %s to %s", member_name, dest_path)
subprocess.check_call(
cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
)
return dest_path
......@@ -95,11 +104,16 @@ CBFS_MAXIMUM_FILE_SIZE = 24 * 2 ** 20 # 24 MiB
def is_header_valid(buf, size, offset=0):
magic, version, romsize, bootblocksize, align, cbfs_offset, architecture, pad = struct.unpack_from(
'!IIIIIIII', buf, offset)
return magic == CBFS_HEADER_MAGIC and \
(version == CBFS_HEADER_VERSION1 or version == CBFS_HEADER_VERSION2) and \
(romsize <= size) and \
(cbfs_offset < romsize)
'!IIIIIIII', buf, offset
)
return (
magic == CBFS_HEADER_MAGIC
and (
version == CBFS_HEADER_VERSION1 or version == CBFS_HEADER_VERSION2
)
and (romsize <= size)
and (cbfs_offset < romsize)
)
class CbfsFile(File):
......@@ -118,7 +132,11 @@ class CbfsFile(File):
# that field is now bound to be little endian
# -- #coreboot, 2015-10-14
rel_offset = struct.unpack('<i', f.read(4))[0]
if rel_offset < 0 and -rel_offset > CBFS_HEADER_SIZE and -rel_offset < size:
if (
rel_offset < 0
and -rel_offset > CBFS_HEADER_SIZE
and -rel_offset < size
):
f.seek(rel_offset, io.SEEK_END)
logger.debug('looking for header at offset: %x', f.tell())
if is_header_valid(f.read(CBFS_HEADER_SIZE), size):
......@@ -127,7 +145,8 @@ class CbfsFile(File):
return False
else:
logger.debug(
'CBFS relative offset seems wrong, scanning whole image')
'CBFS relative offset seems wrong, scanning whole image'
)
f.seek(0, io.SEEK_SET)
offset = 0
buf = f.read(CBFS_HEADER_SIZE)
......
......@@ -32,10 +32,12 @@ class CpioFile(File):
FILE_TYPE_RE = re.compile(r'\bcpio archive\b')
def compare_details(self, other, source=None):
return [Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path,
other.path,
source="file list",
)]
return [
Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path,
other.path,
source="file list",
)
]
......@@ -98,8 +98,11 @@ class DebContainer(LibarchiveContainer):
for name1 in my_members.keys():
main, ext = os.path.splitext(name1)
candidates = [name2 for name2 in other_members.keys() - matched
if os.path.splitext(name2)[0] == main]
candidates = [
name2
for name2 in other_members.keys() - matched
if os.path.splitext(name2)[0] == main
]
if len(candidates) == 1:
yield name1, candidates[0], 0
matched.add(candidates[0])
......@@ -113,8 +116,11 @@ class DebFile(File):
def md5sums(self):
if not hasattr(self, '_md5sums'):
control_tar = self.as_container.control_tar
md5sums_file = control_tar.as_container.lookup_file(
'./md5sums') if control_tar else None
md5sums_file = (
control_tar.as_container.lookup_file('./md5sums')
if control_tar
else None
)
if isinstance(md5sums_file, Md5sumsFile):
self._md5sums = md5sums_file.parse()
else:
......@@ -129,7 +135,8 @@ class DebFile(File):
if not hasattr(self, '_control'):
control_file = self.as_container.control_tar.as_container.lookup_file(
'./control')
'./control'
)
if control_file:
with open(control_file.path, 'rb') as f:
self._control = deb822.Deb822(f)
......@@ -137,21 +144,35 @@ class DebFile(File):
return self._control
def compare_details(self, other, source=None):
return [Difference.from_text_readers(list_libarchive(self.path),
list_libarchive(other.path),
self.path, other.path, source="file list")]
return [
Difference.from_text_readers(
list_libarchive(self.path),
list_libarchive(other.path),
self.path,
other.path,
source="file list",
)
]
class Md5sumsFile(File):
@classmethod
def recognizes(cls, file):
return isinstance(file, ArchiveMember) and \
file.name == './md5sums' and \
isinstance(file.container.source, ArchiveMember) and \
isinstance(file.container.source.container.source, ArchiveMember) and \
DebContainer.RE_CONTROL_TAR.match(file.container.source.container.source.name) and \
isinstance(
file.container.source.container.source.container.source, DebFile)
return (
isinstance(file, ArchiveMember)
and file.name == './md5sums'
and isinstance(file.container.source, ArchiveMember)
and isinstance(
file.container.source.container.source, ArchiveMember
)
and DebContainer.RE_CONTROL_TAR.match(
file.container.source.container.source.name
)
and isinstance(
file.container.source.container.source.container.source,
DebFile,
)
)
def parse(self):
try:
......@@ -171,9 +192,18 @@ class Md5sumsFile(File):
yield " ".join(line.split(" ")[2:])
def compare_details(self, other, source=None):
return [compare_files(self, other, source='md5sums', diff_content_only=True),
Difference.from_text_readers(self.strip_checksum(self.path), self.strip_checksum(other.path),
self.path, other.path, source="line order")]
return [
compare_files(
self, other, source='md5sums', diff_content_only=True
),
Difference.from_text_readers(
self.strip_checksum(self.path),
self.strip_checksum(other.path),
self.path,
other.path,
source="line order",
),
]
class DebTarContainer(TarContainer):
......@@ -184,12 +214,17 @@ class DebTarContainer(TarContainer):
if self.source:
my_md5sums = self.source.container.source.container.source.md5sums
if other.source:
other_md5sums = other.source.container.source.container.source.md5sums
other_md5sums = (
other.source.container.source.container.source.md5sums
)
for my_member, other_member, comment in super().comparisons(other):
if not Config().force_details and \
my_member.name == other_member.name and \
my_md5sums.get(my_member.name, 'my') == other_md5sums.get(other_member.name, 'other'):
if (
not Config().force_details
and my_member.name == other_member.name
and my_md5sums.get(my_member.name, 'my')
== other_md5sums.get(other_member.name, 'other')
):
logger.debug("Skip %s: identical md5sum", my_member.name)
continue
yield my_member, other_member, comment
......@@ -200,12 +235,20 @@ class DebDataTarFile(File):
@classmethod
def recognizes(cls, file):
return isinstance(file, ArchiveMember) and \
isinstance(file.container.source, ArchiveMember) and \
DebContainer.RE_DATA_TAR.match(file.container.source.name) and \
isinstance(file.container.source.container.source, DebFile)
return (
isinstance(file, ArchiveMember)
and isinstance(file.container.source, ArchiveMember)
and DebContainer.RE_DATA_TAR.match(file.container.source.name)
and isinstance(file.container.source.container.source, DebFile)
)
def compare_details(self, other, source=None):
return [Difference.from_text_readers(list_libarchive(self.path, ignore_errors=True),
list_libarchive(other.path, ignore_errors=True),
self.path, other.path, source="file list")]
return [
Difference.from_text_readers(
list_libarchive(self.path, ignore_errors=True),
list_libarchive(other.path, ignore_errors=True),
self.path,
other.path,
source="file list",
)
]
......@@ -51,8 +51,7 @@ class DebControlMember(File):
@property
def path(self):
return os.path.join(
os.path.dirname(self.container.source.path),
self.name,
os.path.dirname(self.container.source.path), self.name
)
def is_directory(self):
......@@ -79,10 +78,9 @@ class DebControlContainer(Container):
if '-' in version:
upstream, revision = version.rsplit('-', 1)
return re.compile(r'_%s(?:-%s)?' % (
re.escape(upstream),
re.escape(revision),
))
return re.compile(
r'_%s(?:-%s)?' % (re.escape(upstream), re.escape(revision))
)
return re.compile(re.escape(version))
......@@ -91,8 +89,9 @@ class DebControlContainer(Container):
yield self._trim_version_number(name), self.get_member(name)
def get_member_names(self):
field = self.source.deb822.get('Files') or \
self.source.deb822.get('Checksums-Sha256')
field = self.source.deb822.get('Files') or self.source.deb822.get(
'Checksums-Sha256'
)
# Show results from debugging packages last; they are rather verbose,
# masking other more interesting differences due to truncating the
......@@ -119,7 +118,9 @@ class DebControlFile(File):
def compare_details(self, other, source=None):
differences = []
for field in sorted(set(self.deb822.keys()).union(set(other.deb822.keys()))):
for field in sorted(
set(self.deb822.keys()).union(set(other.deb822.keys()))
):
if field.startswith('Checksums-') or field == 'Files':
continue
......@@ -131,31 +132,33 @@ class DebControlFile(File):
if field in other.deb822:
other_value = other.deb822.get_as_string(field).lstrip()
differences.append(Difference.from_text(
my_value,
other_value,
self.path,
other.path,
source=field,
))
differences.append(
Difference.from_text(
my_value, other_value, self.path, other.path, source=field
)
)
# Compare Files as string
if self.deb822.get('Files'):
differences.append(Difference.from_text(
self.deb822.get_as_string('Files'),
other.deb822.get_as_string('Files'),
self.path,
other.path,
source='Files',
))
differences.append(
Difference.from_text(
self.deb822.get_as_string('Files'),
other.deb822.get_as_string('Files'),
self.path,
other.path,
source='Files',
)
)
else:
differences.append(Difference.from_text(
self.deb822.get_as_string('Checksums-Sha256'),
other.deb822.get_as_string('Checksums-Sha256'),
self.path,
other.path,
source='Checksums-Sha256',
))
differences.append(
Difference.from_text(
self.deb822.get_as_string('Checksums-Sha256'),
other.deb822.get_as_string('Checksums-Sha256'),
self.path,
other.path,
source='Checksums-Sha256',
)
)
return differences
......@@ -189,13 +192,14 @@ class DotChangesFile(DebControlFile):
files = zip(self.deb822.get('Files'), other.deb822.get('Files'))
files_identical = all(
x == y for x, y in files
if not x['name'].endswith('.buildinfo')
x == y for x, y in files if not x['name'].endswith('.buildinfo')
)
if files_identical and \
len(differences.details) == 1 and \
differences.details[0].source1 == 'Files':
if (
files_identical
and len(differences.details) == 1
and differences.details[0].source1 == 'Files'
):
logger.warning("Ignoring buildinfo file differences")
return None
......@@ -219,8 +223,7 @@ class DotDscFile(DebControlFile):
# XXX: this will not work for containers
in_dsc_path = os.path.join(
os.path.dirname(file.path),
d['Name'],
os.path.dirname(file.path), d['Name']
)
if not os.path.exists(in_dsc_path):
return False
......@@ -277,8 +280,7 @@ class DotBuildinfoFile(DebControlFile):
# XXX: this will not work for containers
in_buildinfo_path = os.path.join(
os.path.dirname(file.path),
d['Name'],
os.path.dirname(file.path), d['Name']
)
if not os.path.exists(in_buildinfo_path):
return False
......
......@@ -28,7 +28,8 @@ class DotChangesFile(TextFile):
if not difference:
return None
difference.add_comment(
'Unable to find Python debian module. Falling back to text comparison.')
'Unable to find Python debian module. Falling back to text comparison.'
)
return difference
......@@ -40,7 +41,8 @@ class DotDscFile(TextFile):
if not difference:
return None
difference.add_comment(
'Unable to find Python debian module. Falling back to text comparison.')
'Unable to find Python debian module. Falling back to text comparison.'
)
return difference
......@@ -52,5 +54,6 @@ class DotBuildinfoFile(TextFile):
if not difference:
return None
difference.add_comment(
'Unable to find Python debian module. Falling back to text comparison.')
'Unable to find Python debian module. Falling back to text comparison.'
)
return difference
......@@ -70,9 +70,15 @@ class Device(File):
super().cleanup()
def compare(self, other, source=None):
with open(self.path) as my_content, \
open(other.path) as other_content:
return Difference.from_text_readers(my_content, other_content, self.name, other.name, source=source, comment="device")
with open(self.path) as my_content, open(other.path) as other_content:
return Difference.from_text_readers(
my_content,
other_content,
self.name,
other.name,
source=source,
comment="device",
)