Skip to content
Commits on Source (13)
......@@ -7,6 +7,14 @@ python:
- "3.5"
- "3.6"
# Travis does not properly support this yet.
# https://github.com/travis-ci/travis-ci/issues/9815
matrix:
include:
- python: 3.7
dist: xenial
sudo: true
before_install:
install: python setup.py install
......
python-ruffus (2.7-2) UNRELEASED; urgency=medium
python-ruffus (2.8.1-1) unstable; urgency=medium
[ Jelmer Vernooij ]
* Use secure copyright file specification URI.
-- Jelmer Vernooij <jelmer@debian.org> Sat, 27 Oct 2018 15:42:31 +0000
[ Andreas Tille ]
* New upstream version
Closes: 915250
* Standards-Version: 4.2.1
* Build-Depends: python-pytest
* d/rules: Test script needs bash options
* Fix autopkgtest (enforce bash, set HOME variable)
-- Andreas Tille <tille@debian.org> Sun, 02 Dec 2018 16:19:09 +0100
python-ruffus (2.7-1) unstable; urgency=medium
......
......@@ -8,12 +8,14 @@ Build-Depends: debhelper (>= 11~),
dh-python,
python-all,
python-setuptools,
python-pytest,
python3-all,
python3-setuptools,
python3-sphinx,
python3-sphinx-rtd-theme,
python3-pytest,
graphviz
Standards-Version: 4.1.5
Standards-Version: 4.2.1
Vcs-Browser: https://salsa.debian.org/med-team/python-ruffus
Vcs-Git: https://salsa.debian.org/med-team/python-ruffus.git
Homepage: http://www.ruffus.org.uk/
......
......@@ -6,7 +6,7 @@ Description: For some strange reason without this patch the test ends up in
--- a/ruffus/test/test_pool_manager.py
+++ b/ruffus/test/test_pool_manager.py
@@ -1,8 +1,15 @@
@@ -1,3 +1,11 @@
+#!/usr/bin/env python
+import os
+import sys
......@@ -18,8 +18,3 @@ Description: For some strange reason without this patch the test ends up in
import contextlib
import random
import unittest
import ruffus
-import os
import shutil
import glob
import tempfile
use_libjs-mathjax.patch
sphinx.ext.pngmath_deprecated.patch
use_png_instead_of_jpg.patch
fix_test.patch
#use_png_instead_of_jpg.patch
#fix_test.patch
privacy.patch
Author: Andreas Tille <tille@debian.org>
Last-Update: Mon, 25 Jul 2016 13:19:50 +0200
Bug-Debian: https://bugs.debian.org/832299
Description: Fix sphinx.ext.mathjax: other math package is already loaded
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -36,7 +36,7 @@ def setup(app):
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo',
- 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
+ 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
mathjax_path = 'file:///usr/share/javascript/mathjax/MathJax.js?config=TeX-AMS-MML_HTMLorMML'
......@@ -6,7 +6,7 @@ Description: Use Debian packaged mathjax
+++ b/doc/conf.py
@@ -38,6 +38,8 @@ def setup(app):
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo',
'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
+mathjax_path = 'file:///usr/share/javascript/mathjax/MathJax.js?config=TeX-AMS-MML_HTMLorMML'
+
......
......@@ -16,7 +16,7 @@ Description: Create PNG instead of JPG to work around bug #827806
--- a/ruffus/task.py
+++ b/ruffus/task.py
@@ -4573,7 +4573,7 @@ def pipeline_printout_graph(stream,
@@ -4051,7 +4051,7 @@ def pipeline_printout_graph(stream,
:param stream: where to print to
:type stream: file-like object with ``write()`` function
......@@ -27,7 +27,7 @@ Description: Create PNG instead of JPG to work around bug #827806
:param target_tasks: targets task functions which will be run if they are
--- a/ruffus/test/test_pipeline_printout_graph.py
+++ b/ruffus/test/test_pipeline_printout_graph.py
@@ -124,7 +124,7 @@ class Test_ruffus(unittest.TestCase):
@@ -114,7 +114,7 @@ class Test_ruffus(unittest.TestCase):
os.makedirs(tempdir)
#
......@@ -35,8 +35,8 @@ Description: Create PNG instead of JPG to work around bug #827806
+ # check graphviz exists for turning dot files into png, svg etc
#
try:
process = Popen("echo what | dot", stdout=PIPE, stderr=STDOUT, shell = True)
@@ -144,7 +144,7 @@ class Test_ruffus(unittest.TestCase):
process = Popen("echo what | dot", stdout=PIPE,
@@ -136,7 +136,7 @@ class Test_ruffus(unittest.TestCase):
print(" Run pipeline normally...")
if self.graph_viz_present:
pipeline_printout_graph(tempdir + "flowchart.dot", pipeline="main")
......@@ -45,8 +45,8 @@ Description: Create PNG instead of JPG to work around bug #827806
target_tasks=[subdivide_start],
forcedtorun_tasks=[split_start],
no_key_legend=True)
@@ -175,7 +175,7 @@ class Test_ruffus(unittest.TestCase):
test_pipeline.subdivide(subdivide_start, split_start, formatter(), tempdir + '{basename[0]}_*.subdivided', tempdir + '{basename[0]}')
@@ -172,7 +172,7 @@ class Test_ruffus(unittest.TestCase):
), tempdir + '{basename[0]}_*.subdivided', tempdir + '{basename[0]}')
if self.graph_viz_present:
test_pipeline.printout_graph(tempdir + "flowchart.dot")
- test_pipeline.printout_graph(tempdir + "flowchart.jpg",
......
......@@ -21,10 +21,19 @@ override_dh_installdocs:
override_dh_auto_test:
ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
cd ruffus/test && \
sh ./run_all_unit_tests.cmd && \
sh ./run_all_unit_tests3.cmd && \
rm -rf .ruffus_history.sqlite __pycache__ *.pyc
dh_auto_test -- --test --system=custom \
--test-args='set -e; \
cd {build_dir}/$(PYBUILD_NAME)/test ; \
if [ "{version.major}" = 2 ] ; then \
ln -s /build/python-ruffus-*/ruffus/test/Makefile ; \
cp -a run_all_unit_tests.cmd /tmp/run_all_unit_tests.cmd ; \
else \
sed "s/^python3/python{version}/" run_all_unit_tests3.cmd > /tmp/run_all_unit_tests.cmd ; \
fi ;\
PYTHONPATH={build_dir} HOME=`mktemp -d` bash /tmp/run_all_unit_tests.cmd'
find . -name .ruffus_history.sqlite -delete
find . -name "*.pyc" -delete
find . -name __pycache__ | xargs rm -rf
endif
override_dh_auto_clean:
......@@ -47,6 +56,7 @@ override_dh_install:
# leave only the test starter that fits Python version
find debian/python3-$(PYBUILD_NAME) -name run_all_unit_tests.cmd -delete
find debian/python-$(PYBUILD_NAME) -name run_all_unit_tests3.cmd -delete
sed -i '1s?^#! */usr/bin/env *python *$$?#!/usr/bin/python3?' `grep -Rl '^#! */usr/bin/env *python *$$' debian/python3-$(PYBUILD_NAME)`
override_dh_fixperms:
dh_fixperms
......
......@@ -13,12 +13,11 @@ find . -type f -name "*.gz" -exec gunzip \{\} \;
# mod +x [a-z]*.py
export LC_ALL=C.UTF-8
export HOME=$ADTTMP
for testscript in run_all_unit_tests*.cmd ; do
# exclude tests with known issues ... should not be needed any more since fix_test.patch
# sed -i -e 's/\(unittest test_file_name_parameters.*\)\\/\1 true/' \
# -e 's/\(unittest test_ruffus_utility.*\)\\/\1 true/' \
# $testscript
sh $testscript
bash $testscript
done
# rm -f $ADTTMP/*
......@@ -36,7 +36,7 @@ def setup(app):
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo',
'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
......
......@@ -20,7 +20,7 @@ Start Here:
gallery.rst
why_ruffus.rst
tutorials/new_tutorial/manual_contents.rst
release.rst
=====================
......
*************
Release notes
*************
=============
Release 2.8.1
=============
* [#101] compatibility with gevent >= 1.2
* add lookup_pipeline to exported functions
* fix tests (thanks @LocutusOfBorg, @xnox)
=============
Release 2.8.0
=============
* Ctrl-C will kill drmaa jobs, SIGUSR1 will suspend jobs and SIGUSR2
will resume.
* [#99] use gevent semaphores
* [#87] run everything through autopep8
* [#86] use pytest for testing
* python3.7 compatibility, thanks to @jbarlow83, @QuLogic
#!/usr/bin/env python
################################################################################
#
# __init__.py
......@@ -28,7 +27,8 @@
from .task import Pipeline, Task
# pipeline functions
from .task import pipeline_printout, pipeline_printout_graph, pipeline_run, pipeline_get_task_names, register_cleanup
from .task import pipeline_printout, pipeline_printout_graph, pipeline_run, pipeline_get_task_names, register_cleanup, \
lookup_pipeline
# decorators
from .task import originate, split, subdivide, transform, merge, collate, follows
......@@ -37,19 +37,22 @@ from .task import originate, split, subdivide, transform, merge, collate, follow
from .task import files, parallel
# filter / indicators
from .task import suffix, regex, formatter, inputs, add_inputs, touch_file
from .task import touch_file
from .file_name_parameters import suffix, regex, formatter, inputs, add_inputs
# deprecated
from .task import files_re, combine
from .task import files_re
from .ruffus_utility import combine
from .task import check_if_uptodate, active_if, jobs_limit, graphviz, mkdir, output_from, posttask
from .task import stderr_logger, black_hole_logger, JobSignalledBreak, runtime_parameter
from .task import check_if_uptodate, active_if, jobs_limit, graphviz, mkdir, posttask
from .ruffus_utility import output_from, runtime_parameter
from .task import stderr_logger, black_hole_logger
from .ruffus_exceptions import JobSignalledBreak
from .graph import graph_colour_demo_printout
from .file_name_parameters import needs_update_check_modify_time
from . import cmdline
from . import combinatorics
# output_dependency_tree_in_dot_format, output_dependency_tree_key_in_dot_format
from . import ruffus_version
__version__ = ruffus_version.__version
......@@ -30,7 +30,6 @@ if sys.hexversion < 0x03000000:
#################################################################################
def adjacent_pairs_iterate(array, reverse=False):
"""
returns pairs of iterators to successive positions
......@@ -60,9 +59,6 @@ def unit_test():
print(i, j)
print(numbers)
if __name__ == '__main__':
unit_test()
#!/usr/bin/env python
################################################################################
#
#
......@@ -24,6 +23,12 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#################################################################################
import sys
from .ruffus_utility import CHECKSUM_REGENERATE
from . import proxy_logger
from . import task
import logging.handlers
import logging
"""
********************************************
......@@ -87,14 +92,15 @@
#
# print options
#
flowchart_formats = ["svg", "svgz", "png", "jpg", "psd", "tif", "eps", "pdf", "dot"]
flowchart_formats = ["svg", "svgz", "png",
"jpg", "psd", "tif", "eps", "pdf", "dot"]
# "jpeg", "gif", "plain", "ps", "wbmp", "canon",
# "cmap", "cmapx", "cmapx_np", "fig", "gd", "gd2",
# "gv", "imap", "imap_np", "ismap", "jpe", "plain-ext",
# "ps2", "tk", "vml", "vmlz", "vrml", "x11", "xdot", "xlib"
# Replace last comma with " and". Mad funky, unreadable reverse replace code: couldn't resist!
flowchart_formats_str = ", ".join(["%r" % ss for ss in flowchart_formats])[::-1].replace(" ,", ", or "[::-1], 1)[::-1]
flowchart_formats_str = ", ".join(["%r" % ss for ss in flowchart_formats])[
::-1].replace(" ,", ", or "[::-1], 1)[::-1]
# _________________________________________________________________________________________
......@@ -139,13 +145,14 @@ def get_argparse (*args, **args_dict):
parser = argparse.ArgumentParser(*args, **args_dict)
return append_to_argparse(parser, **orig_args_dict)
# _________________________________________________________________________________________
# append_to_argparse
# _________________________________________________________________________________________
def append_to_argparse(parser, **args_dict):
"""
Common options:
......@@ -170,7 +177,8 @@ def append_to_argparse (parser, **args_dict):
try:
ignored_args = set(args_dict["ignored_args"])
except:
raise Exception("Error: expected ignored_args = ['list_of', 'option_names']")
raise Exception(
"Error: expected ignored_args = ['list_of', 'option_names']")
else:
ignored_args = set()
......@@ -181,12 +189,12 @@ def append_to_argparse (parser, **args_dict):
action="append",
help="Print more verbose messages for each additional verbose level.")
if "version" not in ignored_args:
common_options.add_argument('--version', action='version', version=prog_version)
common_options.add_argument(
'--version', action='version', version=prog_version)
if "log_file" not in ignored_args:
common_options.add_argument("-L", "--log_file", metavar="FILE", type=str,
help="Name and path of log file")
#
# pipeline
#
......@@ -237,8 +245,6 @@ def append_to_argparse (parser, **args_dict):
metavar="JOBNAME", type=str,
help="Task(s) which will be included even if they are up to date.", default=[])
return parser
......@@ -313,6 +319,8 @@ def get_optparse (*args, **args_dict):
# optparse is deprecated!
# _________________________________________________________________________________________
def append_to_optparse(parser, **args_dict):
"""
Set up OptionParser from optparse
......@@ -348,7 +356,8 @@ def append_to_optparse (parser, **args_dict):
try:
ignored_args = set(args_dict["ignored_args"])
except:
raise Exception("Error: expected ignored_args = ['list_of', 'option_names']")
raise Exception(
"Error: expected ignored_args = ['list_of', 'option_names']")
else:
ignored_args = set()
......@@ -442,20 +451,11 @@ def append_to_optparse (parser, **args_dict):
type="string",
help="Pipeline task(s) which will be included even if they are up to date.")
return parser
import logging
import logging.handlers
MESSAGE = 15
logging.addLevelName(MESSAGE, "MESSAGE")
from . import task
from . import proxy_logger
from .ruffus_utility import CHECKSUM_REGENERATE
import sys
# 88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
......@@ -541,6 +541,7 @@ def setup_logging_factory (logger_name, args):
"""
Ignore INFO messages
"""
def filter(self, record):
return logging.INFO != record.levelno
......@@ -548,6 +549,7 @@ def setup_logging_factory (logger_name, args):
"""
for when there is no logging
"""
def emit(self, record):
pass
......@@ -558,6 +560,7 @@ def setup_logging_factory (logger_name, args):
# log to file if that is specified
if log_file_name:
handler = logging.FileHandler(log_file_name, delay=False)
class stipped_down_formatter(logging.Formatter):
def format(self, record):
prefix = ""
......@@ -570,7 +573,8 @@ def setup_logging_factory (logger_name, args):
else:
self._fmt = " %(asctime)s - %(levelname)-7s - %(message)s"
return prefix + logging.Formatter.format(self, record)
handler.setFormatter(stipped_down_formatter("%(asctime)s - %(name)s - %(levelname)6s - %(message)s", "%H:%M:%S"))
handler.setFormatter(stipped_down_formatter(
"%(asctime)s - %(name)s - %(levelname)6s - %(message)s", "%H:%M:%S"))
handler.setLevel(MESSAGE)
new_logger.addHandler(handler)
has_handler = True
......@@ -595,8 +599,6 @@ def setup_logging_factory (logger_name, args):
return new_logger
#
# valid arguments to each function which are not exposed by any options in the command line
#
......@@ -657,7 +659,6 @@ def get_extra_options_appropriate_for_command (appropriate_option_names, extra_o
return appropriate_options
# _________________________________________________________________________________________
# handle_verbose
......@@ -709,7 +710,8 @@ def handle_verbose (options):
for vv in options.verbose:
mm = match_regex.match(vv)
if not mm:
raise Exception("error: verbosity argument is specified as --verbose INT or --verbose INT:INT. invalid value '%s'" % vv)
raise Exception(
"error: verbosity argument is specified as --verbose INT or --verbose INT:INT. invalid value '%s'" % vv)
if mm.group(1):
curr_verbosity += 1
else:
......@@ -720,7 +722,8 @@ def handle_verbose (options):
# set verbose_abbreviated_path unless set explicity by the user
# in which case we shall prudently not override it!
if not hasattr(options, "verbose_abbreviated_path"):
setattr(options, "verbose_abbreviated_path", curr_verbose_abbreviated_path)
setattr(options, "verbose_abbreviated_path",
curr_verbose_abbreviated_path)
options.verbose = curr_verbosity
#
return options
......@@ -758,7 +761,6 @@ def run (options, **extra_options):
if not hasattr(options, attr_name):
setattr(options, attr_name, None)
#
# handle verbosity specification
#
......@@ -776,7 +778,8 @@ def run (options, **extra_options):
touch_files_only = False
if options.just_print:
appropriate_options = get_extra_options_appropriate_for_command (extra_pipeline_printout_options, extra_options)
appropriate_options = get_extra_options_appropriate_for_command(
extra_pipeline_printout_options, extra_options)
task.pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
history_file=options.history_file,
verbose_abbreviated_path=options.verbose_abbreviated_path,
......@@ -784,7 +787,8 @@ def run (options, **extra_options):
return False
elif options.flowchart:
appropriate_options = get_extra_options_appropriate_for_command (extra_pipeline_printout_graph_options, extra_options)
appropriate_options = get_extra_options_appropriate_for_command(
extra_pipeline_printout_graph_options, extra_options)
task.pipeline_printout_graph(open(options.flowchart, "wb"),
options.flowchart_format,
options.target_tasks,
......@@ -811,14 +815,14 @@ def run (options, **extra_options):
else:
multithread = None
if not "logger" in extra_options:
extra_options["logger"] = None
if extra_options["logger"] == False:
extra_options["logger"] = task.black_hole_logger
elif extra_options["logger"] is None:
extra_options["logger"] = task.stderr_logger
appropriate_options = get_extra_options_appropriate_for_command (extra_pipeline_run_options, extra_options)
appropriate_options = get_extra_options_appropriate_for_command(
extra_pipeline_run_options, extra_options)
task.pipeline_run(options.target_tasks,
options.forced_tasks,
multiprocess=options.jobs,
......@@ -829,5 +833,3 @@ def run (options, **extra_options):
verbose_abbreviated_path=options.verbose_abbreviated_path,
**appropriate_options)
return True
#!/usr/bin/env python
################################################################################
#
#
......@@ -72,11 +71,14 @@ from .task import task_decorator
class product(task_decorator):
pass
class permutations(task_decorator):
pass
class combinations(task_decorator):
pass
class combinations_with_replacement(task_decorator):
pass
#!/usr/bin/python
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
......@@ -90,18 +90,17 @@ __version__ = '1.3.1'
import sqlite3
try:
from collections.abc import MutableMapping
except ImportError:
# MutableMapping is new in Python 2.6+
from collections import MutableMapping
except ImportError:
# DictMixin will be (or is?) deprecated in the Python 3.x series
from UserDict import DictMixin as MutableMapping
from os import path
try:
import cPickle as pickle
except ImportError:
import pickle
import itertools
import sys
class DbDict(MutableMapping):
''' DbDict, a dictionary-like object with SQLite back-end '''
......@@ -114,10 +113,8 @@ class DbDict(MutableMapping):
else:
self.con = sqlite3.connect(filename)
# _____________________________________________________________________________________
# Add automatic pickling and unpickling
def pickle_loads(self, value):
......@@ -127,6 +124,7 @@ class DbDict(MutableMapping):
if self.picklevalues:
value = pickle.loads(bytes(value))
return value
def pickle_dumps(self, value):
"""
pickle.load if specified
......@@ -269,7 +267,8 @@ class DbDict(MutableMapping):
try:
self._update(list(items))
except TypeError:
raise ValueError('Could not interpret value of parameter `items` as a dict, list/tuple or iterator.')
raise ValueError(
'Could not interpret value of parameter `items` as a dict, list/tuple or iterator.')
if kwds:
self._update(list(kwds.items()))
......@@ -340,6 +339,7 @@ class DbDict(MutableMapping):
self.con.execute('reindex sqlite_autoindex_data_1')
self.con.commit()
def dbdict(filename, picklevalues=False):
'''Open a persistent dictionary for reading and writing.
......@@ -351,6 +351,7 @@ def dbdict(filename, picklevalues=False):
'''
return DbDict(filename, picklevalues)
def open(filename, picklevalues=False):
'''Open a persistent dictionary for reading and writing.
......@@ -362,6 +363,7 @@ def open(filename, picklevalues=False):
'''
return DbDict(filename, picklevalues)
if __name__ == '__main__':
# Perform some tests
......@@ -420,7 +422,8 @@ if __name__ == '__main__':
# test remove
d.remove(list(range(8, 10)))
assert len(d.get(list(range(8,10)))) == 0, 'Items not removed successfully'
assert len(d.get(list(range(8, 10)))
) == 0, 'Items not removed successfully'
d.clear()
......
This diff is collapsed.
This diff is collapsed.