CHANGELOG.rst

+[5.5.4] - 2019-07-21
+====================
+Changed
+-------
+- Reports now automatically include workflow code and configuration for improved transparency.
+
 [5.5.3] - 2019-07-11
 ====================
 Changed

debian/README.source

@@ -13,3 +13,17 @@ with in 0011-fix-privacy-breach.patch.  Here are the remaining issues:
      should be packaged to solve the two remaining issues.
 
  -- Andreas Tille <tille@debian.org>  Tue, 12 Dec 2017 11:32:28 +0100
+
+Missing (optional) dependencies:
+
+- conda
+- gfal-copy and other gfal-* commands for gfal/gridftp (missing from gfal2?)
+
+Missing (optional) Python dependencies:
+
+- moto
+- google.cloud (google-cloud-sdk)
+- ftputil
+- pysftp
+- XRootD
+- aioeasywebdav

debian/changelog

-snakemake (5.5.3-2) UNRELEASED; urgency=medium
+snakemake (5.5.4-1) UNRELEASED; urgency=medium
 
+  [ Dylan Aïssi ]
   * Apply patch from Chris Lamb to make the build reproducible
      (Closes: #932116).
 
- -- Dylan Aïssi <daissi@debian.org>  Tue, 16 Jul 2019 08:13:48 +0200
+  [ Michael R. Crusoe ]
+  * New upstream release
+  * debian/control: Add missing build-deps (as shown in errors during the
+                    documenation building)
+  * debian/README.source: Document missing (optional?) dependencies
+  * debian/rules: Fix test invocation, as almost all tests were missed;
+                  Document what is known about the skipped tests
+  * Apply patch from upstream to fix duplicated argument (Closes: #933527)
+
+ -- Michael R. Crusoe <michael.crusoe@gmail.com>  Wed, 31 Jul 2019 11:24:24 +0200
 
 snakemake (5.5.3-1) unstable; urgency=medium
 

debian/control

@@ -6,10 +6,15 @@ Priority: optional
 Build-Depends: debhelper (>= 11~),
                dh-python,
                ca-certificates,
+               cwltool,
+               imagemagick,
                python3,
                python3-boto,
+               python3-botocore,
                python3-configargparse,
                python3-datrie,
+               python3-dropbox,
+               python3-flask,
                python3-git,
                python3-jsonschema,
                python3-networkx,
@@ -29,6 +34,7 @@ Build-Depends: debhelper (>= 11~),
                python3-wrapt,
                python3-yaml,
                r-cran-rmarkdown
+# python3-irodsclient,  # when that enters testing
 Standards-Version: 4.3.0
 Vcs-Browser: https://salsa.debian.org/med-team/snakemake
 Vcs-Git: https://salsa.debian.org/med-team/snakemake.git
@@ -54,8 +60,14 @@ Depends: ${misc:Depends},
          python3-setuptools,
          python3-wrapt,
          python3-yaml
-Recommends: python3-boto,
-            r-cran-rmarkdown
+Recommends: cwltool,
+            python3-boto,
+            python3-botocore,
+            python3-dropbox,
+            python3-flask,
+            r-cran-rmarkdown,
+            imagemagick
+# python3-irodsclient, when it enters testing
 Description: pythonic workflow management system
  Build systems like GNU Make are frequently used to create complicated
  workflows, e.g. in bioinformatics. This project aims to reduce the

debian/docs

+tests/

debian/patches/0012-reproducible-build.patch

@@ -2,9 +2,9 @@ Description: Make the build reproducible
 Author: Chris Lamb <lamby@debian.org>
 Last-Update: 2019-07-15
 
---- snakemake-5.5.3.orig/snakemake/report/__init__.py
-+++ snakemake-5.5.3/snakemake/report/__init__.py
-@@ -98,10 +98,12 @@ def data_uri_from_file(file, defaultenc=
+--- snakemake.orig/snakemake/report/__init__.py
++++ snakemake/snakemake/report/__init__.py
+@@ -100,10 +100,12 @@
  
  
  def report(text, path,
@@ -18,9 +18,9 @@ Last-Update: 2019-07-15
      outmime, _ = mimetypes.guess_type(path)
      if outmime != "text/html":
          raise ValueError("Path to report output has to be an HTML file.")
---- snakemake-5.5.3.orig/snakemake/utils.py
-+++ snakemake-5.5.3/snakemake/utils.py
-@@ -189,7 +189,7 @@ def makedirs(dirnames):
+--- snakemake.orig/snakemake/utils.py
++++ snakemake/snakemake/utils.py
+@@ -189,7 +189,7 @@
  
  
  def report(text, path,
@@ -29,7 +29,7 @@ Last-Update: 2019-07-15
             defaultenc="utf8",
             template=None,
             metadata=None, **files):
-@@ -233,6 +233,8 @@ def report(text, path,
+@@ -233,6 +233,8 @@
          metadata (str):     E.g. an optional author name or email address.
  
      """

debian/patches/0013-remove-duplicate-keyword-argument.patch

+From 6d013348a3501b6c183438cfb44bf78704128925 Mon Sep 17 00:00:00 2001
+From: Alistair Miles <alimanfoo@googlemail.com>
+Date: Mon, 29 Jul 2019 14:53:04 +0000
+Subject: [PATCH] Merged in
+ alimanfoo/snakemake/Alistair-Miles/remove-duplicate-keyword-argument-1563308166092
+ (pull request #397)
+
+remove duplicate keyword argument
+---
+ snakemake/remote/gfal.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+--- snakemake.orig/snakemake/remote/gfal.py
++++ snakemake/snakemake/remote/gfal.py
+@@ -26,7 +26,7 @@
+     supports_default = True
+     allows_directories = True
+ 
+-    def __init__(self, *args, keep_local=False, stay_on_remote=False, is_default=False, stay_on_remote=False, retry=5, **kwargs):
++    def __init__(self, *args, keep_local=False, stay_on_remote=False, is_default=False, retry=5, **kwargs):
+         super(RemoteProvider, self).__init__(*args, keep_local=keep_local, stay_on_remote=stay_on_remote, is_default=is_default, **kwargs)
+         self.retry = retry
+ 

debian/patches/boto3_is_just_boto

+Author: Michael R. Crusoe <michael.crusoe@gmail.com>
+Description: In Debian, boto3 is just boto
+--- snakemake.orig/snakemake/remote/S3.py
++++ snakemake/snakemake/remote/S3.py
+@@ -16,7 +16,7 @@
+ 
+ try:
+     # third-party modules
+-    import boto3
++    import boto as boto3
+     import botocore
+ except ImportError as e:
+     raise WorkflowError("The Python 3 package 'boto3' "
+--- snakemake.orig/snakemake/remote/S3Mocked.py
++++ snakemake/snakemake/remote/S3Mocked.py
+@@ -19,7 +19,7 @@
+ 
+ try:
+     # third-party
+-    import boto3
++    import boto as boto3
+     from moto import mock_s3
+ except ImportError as e:
+     raise WorkflowError("The Python 3 packages 'moto' and boto3' " +
+--- snakemake.orig/tests/test_static_remote/S3MockedForStaticTest.py
++++ snakemake/tests/test_static_remote/S3MockedForStaticTest.py
+@@ -20,7 +20,7 @@
+ 
+ try:
+     # third-party
+-    import boto3
++    import boto as boto3
+     from moto import mock_s3
+     import filechunkio
+ except ImportError as e:

debian/patches/series

@@ -10,3 +10,5 @@
 # 0010-skip-test-without-rmarkdown.patch
 0011-fix-privacy-breach.patch
 0012-reproducible-build.patch
+0013-remove-duplicate-keyword-argument.patch
+boto3_is_just_boto

debian/rules

@@ -6,6 +6,17 @@
 export PYBUILD_NAME=snakemake
 export PYBUILD_DESTDIR_python3=debian/snakemake
 export PYBUILD_BEFORE_TEST_python3=chmod +x {dir}/bin/snakemake; cp -r {dir}/bin {dir}/tests {build_dir}
+export PYBUILD_TEST_ARGS=python{version} -m pytest tests/test*.py -v -k 'not report and not ancient and not test_script and not default_remote and not issue635 and not convert_to_cwl and not issue1083 and not issue1092 and not issue1093'
+
+# test_report
+# test_ancient
+# test_script requires conda; manually disabling conda show the need for the binary 'julia'
+# test_default_remote requires network access
+# test_issue634 requires conda, but passes when conda is turned off manually
+# test_convert_to_cwl tries to build a singularity format software image from docker://quay.io/snakemake/snakemake:v5.5.4
+# test_issue1083 tries to build a singularity format software image from docker://bash
+# test_issue1093 fails due to conda usage; commenting that out and installing bwa produces a different ordering than desired
+
 export PYBUILD_AFTER_TEST_python3=rm -fr {build_dir}/bin {build_dir}/tests
 
 export PATH:=$(shell pybuild --print build_dir --interpreter python3 --name $(PYBUILD_NAME))/bin:$(PATH)
@@ -21,3 +32,6 @@ override_dh_auto_build:
 	PATH=$(shell pybuild --print build_dir --interpreter python3)/bin:$(PATH) \
 	PYTHONPATH=$(shell pybuild --print build_dir --interpreter python3) \
 		python3 setup.py build_sphinx
+
+override_dh_auto_test:
+	PYBUILD_SYSTEM=custom dh_auto_test

debian/source/lintian-overrides

 # This is just pure test data we need to compare with
-snakemake source: source-is-missing tests/test_report/report.html line length is *
 snakemake source: source-is-missing tests/test_report/expected-results/report.html line length is *

docs/getting_started/installation.rst

-.. getting_started-installation:
+.. _getting_started-installation:
 
 ============
 Installation
@@ -7,6 +7,8 @@ Installation
 Snakemake is available on PyPi as well as through Bioconda and also from source code.
 You can use one of the following ways for installing Snakemake.
 
+.. _conda-install:
+
 Installation via Conda
 ======================
 

docs/index.rst

@@ -179,6 +179,7 @@ Please consider to add your own.
    getting_started/installation
    getting_started/examples
    tutorial/tutorial
+   tutorial/short
 
 
 .. toctree::

docs/tutorial/advanced.rst

@@ -203,7 +203,7 @@ We modify the rule ``bwa_map`` accordingly:
   initialization to the DAG phase. In contrast to input functions, these can
   optionally take additional arguments ``input``, ``output``, ``threads``, and ``resources``.
 
-Similar to input and output files, ``params`` can be accessed from the shell command or the Python based ``run`` block (see :ref:`tutorial-report`).
+Similar to input and output files, ``params`` can be accessed from the shell command the Python based ``run`` block, or the script directive (see :ref:`tutorial-script`).
 
 Exercise
 ........

docs/tutorial/basics.rst

@@ -365,8 +365,10 @@ Exercise
    :align: center
 
 
+.. _tutorial-script:
+
 Step 6: Using custom scripts
-::::::::::::::::::::
+::::::::::::::::::::::::::::
 
 Usually, a workflow not only consists of invoking various tools, but also contains custom code to e.g. calculate summary statistics or create plots.
 While Snakemake also allows you to directly :ref:`write Python code inside a rule <.. _snakefiles-rules>`_, it is usually reasonable to move such logic into separate scripts.

environment.yml

@@ -31,3 +31,4 @@ dependencies:
   - xorg-libxrender
   - xorg-libxpm
   - gitpython
+  - pygments

misc/vim/syntax/snakemake.vim

@@ -40,11 +40,15 @@ source $VIMRUNTIME/syntax/python.vim
 " version     = "version" ":" statement
 " run         = "run" ":" ni statement
 " shell       = "shell" ":" stringliteral
+" singularity = "singularity" ":" stringliteral
+" conda       = "conda" ":" stringliteral
+" shadow      = "shadow" ":" stringliteral
+
 
 syn keyword pythonStatement	include workdir onsuccess onerror
 syn keyword pythonStatement	ruleorder localrules configfile
-syn keyword pythonStatement	touch protected temp wrapper
-syn keyword pythonStatement	input output params message threads resources
+syn keyword pythonStatement	touch protected temp wrapper conda shadow
+syn keyword pythonStatement	input output params message threads resources singularity
 syn keyword pythonStatement	version run shell benchmark snakefile log script
 syn keyword pythonStatement	rule subworkflow nextgroup=pythonFunction skipwhite
 

setup.py

@@ -49,7 +49,7 @@ setup(
     install_requires=['wrapt', 'requests', 'ratelimiter', 'pyyaml',
                       'configargparse', 'appdirs', 'datrie', 'jsonschema',
                       'docutils', 'gitpython'],
-    extras_require={"reports": ['jinja2', 'networkx']},
+    extras_require={"reports": ['jinja2', 'networkx', 'pygments']},
     classifiers=
     ["Development Status :: 5 - Production/Stable", "Environment :: Console",
      "Intended Audience :: Science/Research",

snakemake/_version.py

@@ -23,9 +23,9 @@ def get_keywords():
     # setup.py/versioneer.py will grep for the variable names, so they must
     # each be defined on a line of their own. _version.py will just call
     # get_keywords().
-    git_refnames = " (HEAD -> master, tag: v5.5.3)"
-    git_full = "66c61c139e1f439e3f78c09df97927ad54363cc3"
-    git_date = "2019-07-11 18:27:18 +0200"
+    git_refnames = " (tag: v5.5.4)"
+    git_full = "a98c6317d6269c6b339bdca70521bbeb155ba90f"
+    git_date = "2019-07-21 09:16:33 +0200"
     keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
     return keywords
 

snakemake/logging.py

@@ -344,7 +344,7 @@ class Logger:
 
 
 def format_dict(dict, omit_keys=[], omit_values=[]):
-    return ", ".join("{}={}".format(name, value)
+    return ", ".join("{}={}".format(name, str(value))
                      for name, value in dict.items()
                      if name not in omit_keys and value not in omit_values)