Skip to content
Commits on Source (4)
  • Paul Wise's avatar
    Python 3: switch from cmp functions to key functions for sorting versions · de0fd830
    Paul Wise authored 
    cmp functions have been removed in Python 3 and are slower.

Use debian.debian_support.Version wrapper class for key functions
as apt_pkg doesn't appear to have an appropriate class.

Thanks-to: Stuart Prescott <stuart@debian.org>

Traceback (most recent call last):
  File "bin/compare-source-package-list", line 1186, in <module>
    files, patches, links, new = process_sources(source_entries, lists_dir)
  File "bin/compare-source-package-list", line 1164, in process_sources
    actions = check_source_package(source_entry, srcpkg)
  File "bin/compare-source-package-list", line 1113, in check_source_package
    derived_from = find_derived_from(tmp_dir, name, version, dsc_name, dsc_sha1, parts_unmodified)
  File "bin/compare-source-package-list", line 885, in find_derived_from
    possibly_derived_from.sort(cmp=lambda a,b: apt_version_cmp(b[1],a[1]))
TypeError: 'cmp' is an invalid keyword argument for this function

Traceback (most recent call last):
  File "bin/compare-source-package-list", line 1245, in <module>
    data.sort(cmp=lambda a,b: cmp(a[0],b[0]) or apt_version_cmp(a[1],b[1]) or cmp(a[2],b[2]) or apt_version_cmp(a[3],b[3]))
TypeError: 'cmp' is an invalid keyword argument for this function

Traceback (most recent call last):
  File "bin/compare-source-package-list", line 1266, in <module>
    data.sort(cmp=lambda a,b: cmp(a[0],b[0]) or apt_version_cmp(a[1],b[1]) or cmp(a[2],b[2]))
TypeError: 'cmp' is an invalid keyword argument for this function
    de0fd830
  • Paul Wise's avatar
    Python 3: correct uses of bytes and UTF-8 encoding · 0b16d108
    Paul Wise authored 
    Prevents crashes from incompatible types.
Prevents printing Python bytes literals in logs.
    0b16d108
  • Paul Wise's avatar
    Python 3: switch to /usr/bin/python3 · bde4ce05
    Paul Wise authored 
    The code now works with python3.
    bde4ce05
  • Paul Wise's avatar
    Python 3: drop use of the subprocess_setup preexec_fn · d778b448
    Paul Wise authored 
    The default is now to restore default signal handlers in sub-processes.
    d778b448
Show whitespace changes
Inline Side-by-side
bin/aggregate-planet-config
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2012 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......
bin/aggregate-sources-files
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2012 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......
bin/aggregate-sources-patches
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2012 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......
bin/check-package-list
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2011 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......@@ -27,16 +27,10 @@ def remove(file):
except OSError: pass
def dump_list(file_name, list_contents):
f = open(file_name,'wb')
f = open(file_name,'w')
f.write(''.join('%s %s\n' % x for x in list_contents))
f.close()
# http://www.chiark.greenend.org.uk/~cjwatson/blog/python-sigpipe.html
def subprocess_setup():
# Python installs a SIGPIPE handler by default. This is usually not what
# non-Python subprocesses expect.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Setup variables
source_hash_types = {
'MD5': ('Files', 32, 'md5sum'),
......@@ -319,7 +313,7 @@ if sources_source_packages != packages_source_packages:
dump_list('packages_source_packages', packages_source_packages)
cmdline = ['diff', '--unified', 'packages_source_packages', 'sources_source_packages']
stdout = open('diff_source_packages', 'w')
process = subprocess.Popen(cmdline, stdout=stdout, stderr=subprocess.PIPE, preexec_fn=subprocess_setup)
process = subprocess.Popen(cmdline, stdout=stdout, stderr=subprocess.PIPE)
output = process.communicate()[1]
stdout.close()
else:
......@@ -332,7 +326,7 @@ if sources_binary_packages != packages_binary_packages:
dump_list('packages_binary_packages', packages_binary_packages)
cmdline = ['diff', '--unified', 'packages_binary_packages', 'sources_binary_packages']
stdout = open('diff_binary_packages', 'w')
process = subprocess.Popen(cmdline, stdout=stdout, stderr=subprocess.PIPE, preexec_fn=subprocess_setup)
process = subprocess.Popen(cmdline, stdout=stdout, stderr=subprocess.PIPE)
output = process.communicate()[1]
stdout.close()
else:
......
bin/check-sources-list
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2011 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......
bin/compare-planet-heads
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2012 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......
bin/compare-source-package-list
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2011 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......@@ -72,6 +72,7 @@ import signal
import subprocess
import yaml
from debian import deb822, changelog
from debian.debian_support import Version
import apt_pkg
import psycopg2
try: import simplejson as json
......@@ -96,20 +97,6 @@ def symlink(source, link):
try: os.symlink(source, link)
except OSError: pass
# http://www.chiark.greenend.org.uk/~cjwatson/blog/python-sigpipe.html
def subprocess_setup():
# Python installs a SIGPIPE handler by default. This is usually not what
# non-Python subprocesses expect.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# We need to map apt_pkg.version_compare return values to cmp return values
# The documentation is incorrect: https://bugs.debian.org/680891
def apt_version_cmp(a, b):
ret = apt_pkg.version_compare(a, b)
if ret < 0: return -1
elif ret > 0: return 1
else: return 0
# Config
md5_cache_dir = os.path.abspath('../md5-farm')
sha1_cache_dir = os.path.abspath('../sha1-farm')
......@@ -191,8 +178,8 @@ def uncompressed_size(filename):
# *.xz
elif magic == "\xfd7zXZ\x00":
cmdline = ['xz', '--verbose', '--list', filename]
process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
output = process.communicate()[0]
process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = process.communicate()[0].decode('utf-8')
if process.returncode:
logging.warning('xz reported failure to check size of %s:', filename)
logging.warning(output)
......@@ -284,16 +271,16 @@ def global_patch_path(name, version, debian_name, debian_version, type=None):
def convert_lzip_to_gzip(dir, name):
cmdline = ['lzip', '-d', '--', name]
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
output = process.communicate()[0]
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = process.communicate()[0].decode('utf-8')
if process.returncode:
logging.warning('lzip reported failure to decompress %s:', name)
logging.warning(output)
return None
bname = name[0:-3] # Strip off .lz
cmdline = ['gzip', '-1', '--', bname] # gzip -1 to reduce overhead
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
output = process.communicate()[0]
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = process.communicate()[0].decode('utf-8')
if process.returncode:
logging.warning('gzip reported failure to compress %s:', bname)
logging.warning(output)
......@@ -784,14 +771,14 @@ def get_changelog_entries(tmp_dir, dsc_name, dsc_sha1):
# Unpack the source tree
logging.debug('unpacking source package %s', dsc_name)
cmdline = ['dpkg-source', '-x', dsc_name, 'extracted']
process = subprocess.Popen(cmdline, cwd=tmp_dir, env=tmp_environ(tmp_dir), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
output = process.communicate()[0]
process = subprocess.Popen(cmdline, cwd=tmp_dir, env=tmp_environ(tmp_dir), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = process.communicate()[0].decode('utf-8')
if process.returncode:
logging.warning('dpkg-source reported failure to extract %s:', dsc_name)
logging.warning(output)
cmdline = ['ls', '-lR', '--time-style=+']
process = subprocess.Popen(cmdline, cwd=tmp_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
output = process.communicate()[0]
process = subprocess.Popen(cmdline, cwd=tmp_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = process.communicate()[0].decode('utf-8')
logging.warning(output)
rmtree(extract_path)
return None
......@@ -882,15 +869,18 @@ def find_derived_from(tmp_dir, name, version, dsc_name, dsc_sha1, parts_unmodifi
return (entry_name, entry_version)
if possibly_derived_from:
logging.debug('finding closest entry in possibly derived from')
possibly_derived_from.sort(cmp=lambda a,b: apt_version_cmp(b[1],a[1]))
version_v = Version(version)
possibly_derived_from = [(entry_name, Version(entry_version)) for entry_name, entry_version in possibly_derived_from]
possibly_derived_from.sort(reverse=True)
for entry_name, entry_version in possibly_derived_from:
if name == entry_name and apt_version_cmp(version, entry_version) >= 0:
if name == entry_name and version_v > entry_version:
entry_version = str(entry_version)
logging.debug('%s %s is an equal or lower version', entry_name, entry_version)
return (entry_name, entry_version)
entry = possibly_derived_from[-1]
entry_name, entry_version = entry
entry_name, entry_version = possibly_derived_from[-1]
entry_version = str(entry_version)
logging.debug('no lower version numbers, returning next highest version %s %s', entry_name, entry_version)
return entry
return (entry_name, entry_version)
logging.debug('finding closest version number in Debian')
for entry in srcpkg_to_derived_from(name, version):
entry_name, entry_version = entry
......@@ -913,8 +903,8 @@ def create_patch(tmp_dir, dsc_name, dsc_sha1, debian_tmp_dir, debian_dsc_name, d
makedirs(os.path.dirname(path_everything))
cmdline = ['debdiff', '--quiet', '--diffstat', debian_dsc_path, dsc_path]
stdout = open(path_everything, 'w')
process = subprocess.Popen(cmdline, env=tmp_environ(tmp_dir), stdout=stdout, stderr=subprocess.PIPE, preexec_fn=subprocess_setup)
output = process.communicate()[1]
process = subprocess.Popen(cmdline, env=tmp_environ(tmp_dir), stdout=stdout, stderr=subprocess.PIPE)
output = process.communicate()[1].decode('utf-8')
stdout.close()
if process.returncode == 255:
logging.warning('debdiff reported failure %s %s:', debian_dsc_name, dsc_name)
......@@ -922,8 +912,8 @@ def create_patch(tmp_dir, dsc_name, dsc_sha1, debian_tmp_dir, debian_dsc_name, d
cmdline = ['ls', '-lR', '--time-style=+']
for name, dir in (derivative_short_name, tmp_dir), ('Debian', debian_tmp_dir):
logging.warning('dir listing for %s:', name)
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
output = process.communicate()[0]
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = process.communicate()[0].decode('utf-8')
logging.warning(output)
return False
elif process.returncode == 0:
......@@ -936,8 +926,8 @@ def create_patch(tmp_dir, dsc_name, dsc_sha1, debian_tmp_dir, debian_dsc_name, d
cmdline = ['ls', '-lR', '--time-style=+']
for name, dir in (derivative_short_name, tmp_dir), ('Debian', debian_tmp_dir):
logging.warning('dir listing for %s:', name)
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
output = process.communicate()[0]
process = subprocess.Popen(cmdline, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = process.communicate()[0].decode('utf-8')
logging.warning(output)
return False
......@@ -945,15 +935,15 @@ def create_patch(tmp_dir, dsc_name, dsc_sha1, debian_tmp_dir, debian_dsc_name, d
if os.path.exists(path_everything) and not os.path.exists(path_debian):
makedirs(os.path.dirname(path_debian))
cmdline = ['filterdiff', '--include=*/debian/*', path_everything]
filterdiff = subprocess.Popen(cmdline, env=tmp_environ(tmp_dir), stdout=subprocess.PIPE, preexec_fn=subprocess_setup)
filterdiff = subprocess.Popen(cmdline, env=tmp_environ(tmp_dir), stdout=subprocess.PIPE)
filterdiff_output = filterdiff.communicate()[0]
diffstat = subprocess.Popen('diffstat', stdin=subprocess.PIPE, stdout=subprocess.PIPE, preexec_fn=subprocess_setup)
diffstat = subprocess.Popen('diffstat', stdin=subprocess.PIPE, stdout=subprocess.PIPE)
diffstat_output = diffstat.communicate(filterdiff_output)[0]
with open(path_debian, 'wb') as f:
f.write('diffstat of debian/ for %s %s\n' % (os.path.splitext(debian_dsc_name)[0], os.path.splitext(dsc_name)[0]))
f.write('\n')
f.write(('diffstat of debian/ for %s %s\n' % (os.path.splitext(debian_dsc_name)[0], os.path.splitext(dsc_name)[0])).encode('utf-8'))
f.write('\n'.encode('utf-8'))
f.write(diffstat_output)
f.write('\n')
f.write('\n'.encode('utf-8'))
f.write(filterdiff_output)
# Patches > 1MiB are probably not that useful, replace them with a link
......@@ -973,15 +963,15 @@ def check_patch(debian_dsc_sha1, dsc_sha1):
lsdiff_path = sha1_lsdiff_path(debian_dsc_sha1, dsc_sha1)
if os.path.exists(lsdiff_path):
logging.debug('lsdiff cache exists for %s', patch_path)
with open(lsdiff_path) as f:
with open(lsdiff_path, encoding='utf-8') as f:
lsdiff = f.read()
else:
logging.debug('lsdiff cache does not exist for %s', patch_path)
cmdline = ['lsdiff', patch_path]
process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, preexec_fn=subprocess_setup)
lsdiff = process.communicate()[0]
process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
lsdiff = process.communicate()[0].decode('utf-8')
makedirs(os.path.dirname(lsdiff_path))
with open(lsdiff_path,'wb') as f:
with open(lsdiff_path,'w') as f:
f.write(lsdiff)
lsdiff = lsdiff.splitlines()
for line in lsdiff:
......@@ -1238,7 +1228,7 @@ filename = sys.argv[5]
data = links
if data:
data = list(set(data))
data.sort(cmp=lambda a,b: cmp(a[0],b[0]) or apt_version_cmp(a[1],b[1]) or cmp(a[2],b[2]) or apt_version_cmp(a[3],b[3]))
data.sort(key=lambda entry: (entry[0], Version(entry[1]), entry[2], Version(entry[3])))
output_data = {}
for debian_name, debian_version, name, version, dsc_url in data:
if debian_name not in output_data:
......@@ -1259,7 +1249,7 @@ filename = sys.argv[6]
data = new
if data:
data = list(set(data))
data.sort(cmp=lambda a,b: cmp(a[0],b[0]) or apt_version_cmp(a[1],b[1]) or cmp(a[2],b[2]))
data.sort(key=lambda entry: (entry[0], Version(entry[1]), entry[2]))
output_data = {}
for name, version, dsc_url in data:
if name not in output_data:
......
bin/derivative-to-planet-config
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2012 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......
bin/logo-to-planet-head
View file @ d778b448
#!/usr/bin/python
#!/usr/bin/python3
# Copyright 2012 Paul Wise
# Released under the MIT/Expat license, see doc/COPYING
......