Skip to content
Commits on Source (2)
  • Thomas Goirand's avatar
    Fix the rest of the files for Python 3. · 0b47bf88
    Thomas Goirand authored
    0b47bf88
  • Thomas Goirand's avatar
    * Switch to package to Python 3: · 336d7650
    Thomas Goirand authored and Holger Levsen's avatar Holger Levsen committed
        - Replaced python- with python3- in debian/control.
        - Removed all python2 (>= 2.7) by a ${python3:Depends}.
        - Replaced python2 by python3 in debian/rules.
        - Replaced dh_python2 stuff by dh_python3 in debian/rules.
        - Patched piupartslib/conf.py, piupartslib/dependencyparser.py,
          piupartslib/packagesdb.py, piupartslib/pkgsummary.py, tests/unittests.py
          so that they are Python 3 compatible.
        - Removed python-lzma (build-)depends as it's in standard in Py 3.
        - Add python3-six as (build-)depends.
        - Fix Makefile to use python3 and not python2.
        - d/piuparts-common.install: switch to python 3.
        - d/rules: remove all __pycache__ folder before proceeding to install.
        - Add --shebang=/usr/bin/python3 when calling dh_python3.
        - Do not use -N piuparts-master -N piuparts-slave when calling dh_python3
          for the 2nd time, so that it correctly adds the interpreter as depends.
        - Lintian overrides "python-script-but-no-python-dep": that's ok, because
          piuparts-common already has the dependency.
    336d7650
......@@ -6,7 +6,7 @@ man1dir = $(mandir)/man1
man8dir = $(mandir)/man8
libdir = $(prefix)/lib
docdir = $(prefix)/share/doc/piuparts
site27 = $(libdir)/python2.7/dist-packages
site3 = $(libdir)/python3/dist-packages
htdocsdir = $(sharedir)/piuparts/htdocs
etcdir = $(prefix)/etc
......@@ -45,7 +45,7 @@ all: build
python_scripts = $(wildcard *.py piupartslib/*.py master-bin/*.py slave-bin/*.py)
python-syntax-check:
@set -e -x; $(foreach py,$(python_scripts),python -m py_compile $(py);)
@set -e -x; $(foreach py,$(python_scripts),python3 -m py_compile $(py);)
$(RM) $(python_scripts:=c)
build: build-stamp build-master-stamp
......@@ -64,19 +64,19 @@ build-master-stamp:
build-doc: $(DOCS_GENERATED)
docs/build: docs/build
sphinx-build docs/ docs/build/
python3 -m sphinx docs/ docs/build/
piuparts.1: docs/piuparts/piuparts.1.txt
sphinx-build -b man -c docs/piuparts/ docs/piuparts/ ./
python3 -m sphinx -b man -c docs/piuparts/ docs/piuparts/ ./
piuparts_slave_run.8: docs/piuparts_slave_run/piuparts_slave_run.8.txt
sphinx-build -b man -c docs/piuparts_slave_run/ docs/piuparts_slave_run/ ./
python3 -m sphinx -b man -c docs/piuparts_slave_run/ docs/piuparts_slave_run/ ./
piuparts_slave_join.8: docs/piuparts_slave_join/piuparts_slave_join.8.txt
sphinx-build -b man -c docs/piuparts_slave_join/ docs/piuparts_slave_join/ ./
python3 -m sphinx -b man -c docs/piuparts_slave_join/ docs/piuparts_slave_join/ ./
piuparts_slave_stop.8: docs/piuparts_slave_stop/piuparts_slave_stop.8.txt
sphinx-build -b man -c docs/piuparts_slave_stop/ docs/piuparts_slave_stop/ ./
python3 -m sphinx -b man -c docs/piuparts_slave_stop/ docs/piuparts_slave_stop/ ./
install-doc: build-stamp
# txt
......@@ -148,8 +148,8 @@ install-conf-4-running-from-git: build-stamp
install -m 0755 update-piuparts-master-setup $(DESTDIR)$(sharedir)/piuparts/master/
install-common: build-stamp
install -d $(DESTDIR)$(site27)/piupartslib
install -m 0644 piupartslib/*.py $(DESTDIR)$(site27)/piupartslib/
install -d $(DESTDIR)$(site3)/piupartslib
install -m 0644 piupartslib/*.py $(DESTDIR)$(site3)/piupartslib/
install -d $(DESTDIR)$(sharedir)/piuparts/lib
install -m 0644 lib/*.sh $(DESTDIR)$(sharedir)/piuparts/lib/
......@@ -201,7 +201,7 @@ install: install-master install-slave
check:
nosetests --verbose
python3 -m nose --verbose
clean:
rm -f build-stamp
......
......@@ -13,6 +13,26 @@ piuparts (1.0.2) UNRELEASED; urgency=medium
[ David Prévot ]
* Use the same favicon as the one from www.d.o.
[ Thomas Goirand ]
* Switch to package to Python 3:
- Replaced python- with python3- in debian/control.
- Removed all python2 (>= 2.7) by a ${python3:Depends}.
- Replaced python2 by python3 in debian/rules.
- Replaced dh_python2 stuff by dh_python3 in debian/rules.
- Patched piupartslib/conf.py, piupartslib/dependencyparser.py,
piupartslib/packagesdb.py, piupartslib/pkgsummary.py, tests/unittests.py
so that they are Python 3 compatible.
- Removed python-lzma (build-)depends as it's in standard in Py 3.
- Add python3-six as (build-)depends.
- Fix Makefile to use python3 and not python2.
- d/piuparts-common.install: switch to python 3.
- d/rules: remove all __pycache__ folder before proceeding to install.
- Add --shebang=/usr/bin/python3 when calling dh_python3.
- Do not use -N piuparts-master -N piuparts-slave when calling dh_python3
for the 2nd time, so that it correctly adds the interpreter as depends.
- Lintian overrides "python-script-but-no-python-dep": that's ok, because
piuparts-common already has the dependency.
-- Holger Levsen <holger@debian.org> Sun, 07 Jul 2019 17:15:41 +0200
piuparts (1.0.1) unstable; urgency=medium
......
......@@ -9,16 +9,16 @@ Rules-Requires-Root: no
Build-Depends:
debhelper-compat (= 11),
dh-python,
python (>= 2.7),
python-debian,
python-apt,
python-distro-info,
python-nose,
python-debianbts,
python-yaml,
python-mox3,
python-lzma,
python-sphinx,
python3-all,
python3-apt,
python3-debian,
python3-debianbts,
python3-distro-info,
python3-mox3,
python3-nose,
python3-six,
python3-sphinx,
python3-yaml,
golang-any,
Build-Depends-Indep:
git,
......@@ -38,8 +38,8 @@ Depends:
lsb-release,
lsof,
mount,
python-debian,
${python:Depends},
python3-debian,
${python3:Depends},
${misc:Depends},
Recommends:
adequate
......@@ -66,19 +66,18 @@ Depends:
# keep this list in sync with piuparts-master-from-git-deps
adduser,
openssh-server,
python-debianbts,
python-setproctitle,
python3-debianbts,
python3-setproctitle,
tango-icon-theme,
xz-utils,
python-yaml,
python (>= 2.7),
${python:Depends},
python3-yaml,
${python3:Depends},
${shlibs:Depends},
${misc:Depends},
Recommends:
apache2 | httpd,
ghostscript,
python-rpy2,
python3-rpy2,
r-base-dev,
r-recommended,
devscripts,
......@@ -100,20 +99,19 @@ Depends:
# this list is synced from piuparts-master
adduser,
openssh-server,
python-debianbts,
python-setproctitle,
python3-debianbts,
python3-setproctitle,
tango-icon-theme,
xz-utils,
python-yaml,
python (>= 2.7),
python3-yaml,
# selected packages from piuparts-master Recommends
apache2 | httpd,
python-rpy2,
python3-rpy2,
# this list is synced from piuparts-common
python-apt,
python-distro-info,
python-lzma,
python3-apt,
python3-distro-info,
${misc:Depends},
${python3:Depends},
Description: dependencies for running piuparts master from git
piuparts is meant as a quality assurance tool for people who create .deb
packages to test them before they upload them to the Debian package archive.
......@@ -133,9 +131,8 @@ Depends:
openssh-client,
screen,
sudo,
python (>= 2.7),
${python:Depends},
${misc:Depends},
${python3:Depends},
Suggests:
schroot
Description: piuparts slave components
......@@ -156,7 +153,6 @@ Depends:
openssh-client,
screen,
sudo,
python (>= 2.7),
# this list is synced from piuparts
debootstrap (>= 1.0.89~),
debsums (>= 2.2.2~),
......@@ -164,12 +160,12 @@ Depends:
lsb-release,
lsof,
mount,
python-debian,
python3-debian,
# this list is synced from piuparts-common
python-apt,
python-distro-info,
python-lzma,
python3-apt,
python3-distro-info,
${misc:Depends},
${python3:Depends},
Description: dependencies for running piuparts slave from git
piuparts is meant as a quality assurance tool for people who create .deb
packages to test them before they upload them to the Debian package archive.
......@@ -183,11 +179,11 @@ Package: piuparts-common
Architecture: all
Depends:
# keep this list in sync with piuparts-{master,slave}-from-git-deps
python-apt,
python-distro-info,
python-lzma,
${python:Depends},
python3-apt,
python3-distro-info,
python3-six,
${misc:Depends},
${python3:Depends},
Description: common piuparts components
piupartslib, common library used by piuparts-master, -report, -analyze and
-slave.
......
etc/piuparts/distros.conf
usr/lib/python2.7/*
usr/lib/python*/*
usr/share/piuparts/lib/*
# It's ok, because it depends on piuparts-common that has the dependency.
piuparts-master: python-script-but-no-python-dep usr/share/piuparts/master/detect_well_known_errors #!/usr/bin/python3
piuparts-master: python-script-but-no-python-dep usr/share/piuparts/piuparts-analyze #!/usr/bin/python3
piuparts-master: python-script-but-no-python-dep usr/share/piuparts/piuparts-master-backend #!/usr/bin/python3
piuparts-master: python-script-but-no-python-dep usr/share/piuparts/piuparts-report #!/usr/bin/python3
# It's ok, because it depends on piuparts-common that has the dependency.
piuparts-slave: python-script-but-no-python-dep usr/share/piuparts/piuparts-slave #!/usr/bin/python3
......@@ -4,15 +4,16 @@
#export DH_VERBOSE=1
%:
dh $@ --with python2
dh $@ --with python3
override_dh_auto_build:
$(MAKE) prefix=/usr build build-doc
override_dh_auto_install:
for i in $$(find . -type d -iname __pycache__) ; do rm -rf $$i ; done
$(MAKE) DESTDIR=$(CURDIR)/debian/tmp prefix=/usr etcdir=/etc install install-doc install-conf
override_dh_python2:
dh_python2 -p piuparts-master -p piuparts-slave /usr/share/piuparts
dh_python2 -N piuparts-master -N piuparts-slave
override_dh_python3:
dh_python3 -p piuparts-master -p piuparts-slave /usr/share/piuparts --shebang=/usr/bin/python3
dh_python3 -N piuparts-master -N piuparts-slave
......@@ -30,12 +30,13 @@ import os
import fcntl
import time
import random
from urllib2 import URLError
import piupartslib
from piupartslib.packagesdb import LogfileExists
from piupartslib.conf import MissingSection
from six.moves.urllib.error import URLError
CONFIG_FILE = "/etc/piuparts/piuparts.conf"
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
......
......@@ -38,7 +38,6 @@ import hashlib
import pickle
import random
import fcntl
from urllib2 import HTTPError, URLError
from collections import deque
# if python-rpy2 ain't installed, we don't draw fancy graphs
......@@ -53,6 +52,8 @@ from piupartslib.conf import MissingSection
from piupartslib.dwke import *
import piupartslib.pkgsummary as pkgsummary
from six.moves.urllib.error import HTTPError, URLError
CONFIG_FILE = "/etc/piuparts/piuparts.conf"
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
......@@ -1367,9 +1368,9 @@ class Section:
"rows": rows,
})
if state == "failed-testing":
count_bugged = string.count(rows, '"bugged/')
count_affected = string.count(rows, '"affected/')
count_failed = string.count(rows, '"fail/')
count_bugged = rows.count('"bugged/')
count_affected = rows.count('"affected/')
count_failed = rows.count('"fail/')
sep = ": "
if count_bugged > 0:
substats += sep + "%s bugged" % count_bugged
......@@ -1380,7 +1381,7 @@ class Section:
if count_failed > 0:
substats += sep + "<span class=\"needs-bugging\">%s failed</span>" % count_failed
else:
count_passed = string.count(rows, '"pass/')
count_passed = rows.count('"pass/')
if count_passed > 0:
substats += ": %s passed" % count_passed
link += "<li><a href=%s>%s</a>%s</li>\n" % \
......
......@@ -40,7 +40,6 @@ import time
import logging
import optparse
import sys
import commands
import tempfile
import shutil
import os
......@@ -51,7 +50,6 @@ import json
import pickle
import subprocess
import traceback
import urllib
import uuid
import apt_pkg
import pipes
......@@ -65,6 +63,11 @@ except ImportError:
import piupartslib.conf
from six.moves import urllib
import six
apt_pkg.init_system()
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
......@@ -1537,24 +1540,24 @@ class Chroot:
changes = diff_selections(self, selections)
deps = {}
nondeps = {}
for name, state_version in iter(changes.items()):
for name, state_version in six.iteritems(changes):
if name in packages:
nondeps[name] = state_version
else:
deps[name] = state_version
deps_to_remove = [name for name, (state, version) in iter(deps.items())
deps_to_remove = [name for name, (state, version) in six.iteritems(deps)
if state == "remove"]
deps_to_purge = [name for name, (state, version) in iter(deps.items())
deps_to_purge = [name for name, (state, version) in six.iteritems(deps)
if state == "purge"]
nondeps_to_remove = [name for name, (state, version) in iter(nondeps.items())
nondeps_to_remove = [name for name, (state, version) in six.iteritems(nondeps)
if state == "remove"]
nondeps_to_purge = [name for name, (state, version) in iter(nondeps.items())
nondeps_to_purge = [name for name, (state, version) in six.iteritems(nondeps)
if state == "purge"]
all_to_remove = deps_to_remove + deps_to_purge + nondeps_to_remove + nondeps_to_purge
all_to_install = [(name, version) for name, (state, version) in iter(deps.items())
all_to_install = [(name, version) for name, (state, version) in six.iteritems(deps)
if state == "install"]
all_to_install += [(name, version) for name, (state, version) in iter(nondeps.items())
all_to_install += [(name, version) for name, (state, version) in six.iteritems(nondeps)
if state == "install"]
# First remove all packages (and reinstall missing ones).
......@@ -1769,7 +1772,7 @@ class Chroot:
dev_ptmx_rel_path = self.relative("dev/ptmx")
if not os.path.islink(dev_ptmx_rel_path):
if not os.path.exists(dev_ptmx_rel_path):
os.mknod(dev_ptmx_rel_path, 0666 | stat.S_IFCHR, os.makedev(5, 2))
os.mknod(dev_ptmx_rel_path, 0o0666 | stat.S_IFCHR, os.makedev(5, 2))
self.mount(self.relative("dev/pts/ptmx"), "/dev/ptmx", opts="bind", no_mkdir=True)
p = subprocess.Popen(["tty"], stdout=subprocess.PIPE,
universal_newlines=True)
......@@ -1778,7 +1781,7 @@ class Chroot:
if p.returncode == 0 and os.path.exists(current_tty):
dev_console = self.relative("/dev/console")
if not os.path.exists(dev_console):
os.mknod(dev_console, 0600, os.makedev(5, 1))
os.mknod(dev_console, 0o0600, os.makedev(5, 1))
self.mount(current_tty, "/dev/console", opts="bind", no_mkdir=True)
self.mount("tmpfs", "/dev/shm", fstype="tmpfs", opts="size=65536k")
if selinux_enabled():
......@@ -2048,8 +2051,8 @@ def diff_meta_data(tree1, tree2, quiet=False):
del tree1_c[name]
del tree2_c[name]
removed = [x for x in iter(tree1_c.items())]
new = [x for x in iter(tree2_c.items())]
removed = [x for x in six.iteritems(tree1)]
new = [x for x in six.iteritems(tree2)]
# fix for #586793
# prune rc?.d symlinks renamed by insserv
......@@ -2120,13 +2123,13 @@ def diff_selections(chroot, selections):
set to to restore original selections."""
changes = {}
current = chroot.get_selections()
for name, (value, version) in iter(current.items()):
for name, (value, version) in six.iteritems(current):
if name not in selections:
changes[name] = ("purge", None)
elif selections[name][0] != value and \
selections[name][0] in ["purge", "install"]:
changes[name] = selections[name]
for name, (value, version) in iter(selections.items()):
for name, (value, version) in six.iteritems(selections):
if name not in current or \
current[name][1] != version:
changes[name] = selections[name]
......@@ -2511,8 +2514,8 @@ def install_and_upgrade_between_distros(package_files, packages_qualified):
if chroot_state is not None:
if chroot.initial_selections != chroot_state["initial_selections"]:
logging.warn("Initial package selections do not match - ignoring loaded reference chroot state")
refsel = [(s, p, v) for p, (s, v) in iter(chroot_state["initial_selections"].items())]
cursel = [(s, p, v) for p, (s, v) in iter(chroot.initial_selections.items())]
refsel = [(s, p, v) for p, (s, v) in six.iteritems(chroot_state["initial_selections"])]
cursel = [(s, p, v) for p, (s, v) in six.iteritems(chroot.initial_selections)]
rsel = [x for x in refsel if not x in cursel]
csel = [x for x in cursel if not x in refsel]
[logging.debug(" -%s" % " ".join(x)) for x in rsel]
......
......@@ -20,12 +20,12 @@
import bz2
import lzma
import zlib
import urllib2
import conf
import dependencyparser
import packagesdb
from piupartslib import dependencyparser
from piupartslib import packagesdb
from six.moves import urllib
class DecompressedStream():
......@@ -82,8 +82,8 @@ def open_packages_url(url):
socket = None
for ext in ['.xz', '.bz2', '.gz', '']:
try:
socket = urllib2.urlopen(url + ext)
except urllib2.HTTPError as httperror:
socket = urllib.request.urlopen(url + ext)
except urllib.error.HTTPError as httperror:
pass
else:
break
......
......@@ -23,8 +23,11 @@
# in a configuration file, that's why).
#
import ConfigParser
import UserDict
try:
import UserDict
except ImportError:
from collections import UserDict
import subprocess
import collections
import re
......@@ -32,6 +35,11 @@ import distro_info
from functools import reduce
import six
from six.moves import configparser
from six.moves import reduce
class MissingSection(Exception):
def __init__(self, filename, section):
......@@ -46,18 +54,18 @@ class MissingMandatorySetting(Exception):
(key, filename),
class Config(UserDict.UserDict):
class Config(UserDict):
def __init__(self, section, defaults, mandatory=[], defaults_section=None):
UserDict.UserDict.__init__(self)
UserDict.__init__(self)
self._section = section
self._defaults_section = defaults_section
for key, value in defaults.iteritems():
for key, value in six.iteritems(defaults):
self[key] = value
self._mandatory = mandatory
def read(self, filename):
cp = ConfigParser.ConfigParser()
cp = configparser.ConfigParser()
cp.read(filename)
if not cp.has_section(self._section):
raise MissingSection(filename, self._section)
......@@ -113,7 +121,7 @@ class Config(UserDict.UserDict):
])
# add mappings for e.g. "oldstable" -> "oldstable"
distmap.update(dict([(val, val) for key, val in distmap.iteritems()]))
distmap.update(dict([(val, val) for key, val in six.iteritems(distmap)]))
# map e.g. "Debian6" -> "oldstable" where debdist.old(result="fullname")
# currently returns 'Debian 6.0 "Squeeze"'
......@@ -148,10 +156,10 @@ class Config(UserDict.UserDict):
return self["arch"]
class DistroConfig(UserDict.UserDict):
class DistroConfig(UserDict):
def __init__(self, filename, mirror):
UserDict.UserDict.__init__(self)
UserDict.__init__(self)
self._mirror = mirror
self._defaults = {
"uri": None,
......@@ -161,7 +169,7 @@ class DistroConfig(UserDict.UserDict):
"depends": None,
"candidates": None,
}
cp = ConfigParser.SafeConfigParser()
cp = configparser.SafeConfigParser()
cp.read(filename)
for section in cp.sections():
self[section] = dict(self._defaults)
......
......@@ -58,14 +58,14 @@ class _Cursor:
def skip_whitespace(self):
while self._pos < self._len and self._input[self._pos].isspace():
self.next()
self.mynext()
def at_end(self):
"""Are we at the end of the input?"""
self.skip_whitespace()
return self._pos >= self._len
def next(self):
def mynext(self):
"""Move to the next character"""
if self._pos < self._len:
self._pos += 1
......@@ -263,7 +263,7 @@ class DependencyParser:
def _parse_version_dependency(self):
self._cursor.skip_whitespace()
if self._cursor.get_char() == "(":
self._cursor.next()
self._cursor.mynext(self._cursor)
self._cursor.skip_whitespace()
opm = self._cursor.match(self._op_pat)
......@@ -285,7 +285,7 @@ class DependencyParser:
self._cursor.skip_whitespace()
if self._cursor.get_char() != ")":
raise DependencySyntaxError("Expected ')'", self._cursor)
self._cursor.next()
self._cursor.mynext(self._cursor)
return opm.group(), verm.group()
else:
......@@ -296,13 +296,13 @@ class DependencyParser:
def _parse_arch_restriction(self):
self._cursor.skip_whitespace()
if self._cursor.get_char() == "[":
self._cursor.next()
self.mynext(self._cursor)
vlist = []
while True:
self._cursor.skip_whitespace()
if self._cursor.get_char() == "]":
self._cursor.next()
self._cursor.mynext(self._cursor)
break
m = self._cursor.match(self._arch_pat)
if not m:
......
......@@ -32,12 +32,18 @@ import random
import stat
import tempfile
import time
import UserDict
try:
import UserDict
except ImportError:
from collections import UserDict
import apt_pkg
import piupartslib
from piupartslib.dependencyparser import DependencyParser
import six
apt_pkg.init_system()
......@@ -54,10 +60,10 @@ def rfc822_like_header_parse(input):
return headers
class Package(UserDict.UserDict):
class Package(UserDict):
def __init__(self, headers):
UserDict.UserDict.__init__(self)
UserDict.__init__(self)
self.headers = headers
for header in headers:
name, value = header.split(":", 1)
......@@ -163,10 +169,10 @@ class Package(UserDict.UserDict):
output_file.write("".join(self.headers))
class PackagesFile(UserDict.UserDict):
class PackagesFile(UserDict):
def __init__(self):
UserDict.UserDict.__init__(self)
UserDict.__init__(self)
self._urllist = []
def load_packages_urls(self, urls, restrict_packages=None):
......@@ -643,7 +649,7 @@ class PackagesDB:
self._in_state[state] = []
todo = []
for package_name, package in self._packages.iteritems():
for package_name, package in six.iteritems(self._packages):
state = self._lookup_package_state(package, use_cached_success, check_outdated)
assert state in self._states
self._package_state[package_name] = state
......@@ -809,7 +815,7 @@ class PackagesDB:
if not providers:
return package_state
states = [self.get_package_state(name, resolve_virtual=False, recurse=recurse) for name in [package_name] + providers]
for state in self._good_states + self._propagate_waiting_state.keys() + self._propagate_error_state.keys():
for state in self._good_states + list(self._propagate_waiting_state.keys()) + self._propagate_error_state.keys():
if state in states:
return state
return package_state
......
......@@ -85,6 +85,9 @@ import datetime
from collections import namedtuple, defaultdict
import six
class SummaryException(Exception):
pass
......@@ -117,7 +120,7 @@ flaginfo = {
]),
}
state2flg = dict([(y, x[0]) for x in flaginfo.iteritems() for y in x[1].states])
state2flg = dict([(y, x[0]) for x in six.iteritems(flaginfo) for y in x[1].states])
def worst_flag(*flags):
......
# -*- coding: utf-8 -*-
import os
import StringIO
import unittest
import piupartslib.packagesdb
import six
class FakeLogDB(piupartslib.packagesdb.LogDB):
"""A fake version of the LogDB class, for testing
......@@ -50,7 +52,7 @@ class PackagesDbTests(unittest.TestCase):
def new_db(self, packages_file_contents):
db = piupartslib.packagesdb.PackagesDB(FakeLogDB())
db.read_packages_file(StringIO.StringIO(packages_file_contents))
db.read_packages_file(six.StringIO(packages_file_contents))
return db
def reserve(self, packages_file_contents):
......