.circleci/config.yml

@@ -10,7 +10,7 @@ jobs:
       - checkout
       - restore_cache:
           keys:
-            - snakemake-{{ checksum ".circleci/setup.sh" }}-{{ checksum "test-environment.yml" }}-{{ checksum ".circleci/common.sh" }}
+            - snakemake-{{ .Environment.CACHE_VERSION }}-{{ checksum ".circleci/setup.sh" }}-{{ checksum "test-environment.yml" }}-{{ checksum ".circleci/common.sh" }}
       - run:
           name: Update PATH
           command: echo 'export PATH="`pwd`/miniconda/bin:$PATH"' >> $BASH_ENV
@@ -18,7 +18,7 @@ jobs:
           name: Setup Conda
           command: .circleci/setup.sh
       - save_cache:
-          key: snakemake-{{ checksum ".circleci/setup.sh" }}-{{ checksum "test-environment.yml" }}-{{ checksum ".circleci/common.sh" }}
+          key: snakemake-{{ .Environment.CACHE_VERSION }}-{{ checksum ".circleci/setup.sh" }}-{{ checksum "test-environment.yml" }}-{{ checksum ".circleci/common.sh" }}
           paths:
             - miniconda
       - run:
@@ -60,7 +60,7 @@ jobs:
             $GCLOUD config set project snakemake-testing
       - run:
           name: Run tests
-          no_output_timeout: 30m
+          no_output_timeout: 60m
           command: |
             export GCLOUD_CLUSTER=t-`uuidgen`
             export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcloud-service-key.json

.readthedocs.yml

+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  version: 3.6
+  install:
+    - requirements: docs/requirements.txt

CHANGELOG.rst

+[5.5.3] - 2019-07-11
+====================
+Changed
+-------
+- Various bug fixes.
+- Polished reports.
+
+[5.5.2] - 2019-06-25
+====================
+Changed
+-------
+- Various minor bug fixes in reports.
+- Speed improvements when using checkpoints.
+
+[5.5.1] - 2019-06-18
+====================
+Changed
+-------
+- Improved report interface. In particular for large files.
+- Small TSV tables are automatically rendered as HTML with datatables.
+- Be more permissive with Snakefile choices: allow "Snakefile", "snakefile", "workflow/Snakefile", "workflow/snakefile". 
+
+[5.5.0] - 2019-05-31
+====================
+Added
+-----
+- Script directives now also support Julia.
+Changed
+-------
+- Various small bug fixes.
+
+[5.4.5] - 2019-04-12
+====================
+
+Changed
+-------
+- Fixed a bug with pipe output.
+- Cleaned up error output.
+
+[5.4.4] - 2019-03-22
+====================
+
+Changed
+-------
+- Vastly improved performance of HTML reports generated with --report, via a more efficient encoding of dara-uri based download links.
+- Tighter layout, plus thumbnails and a lightbox for graphical results in HTML reports.
+- Bug fix for pipe groups.
+- Updated docs.
+- Better error handling in DRMAA executor.
+
+[5.4.3] - 2019-03-11
+====================
+
+Changed
+-------
+- More robust handling of conda environment activation that should work with all setups where the conda is available when starting snakemake.
+- Fixed bugs on windows.
+
+[5.4.2] - 2019-02-15
+====================
+
+Changed
+-------
+- Fixed a bug where git module cannot be imported from wrapper.
+
+[5.4.1] - 2019-02-14
+====================
+
+Added
+-----
+- Warning when R script is used in combination with conda and R_LIBS environment variable is set. This can cause unexpected results and should be avoided.
+
+Changed
+-------
+- Improved quoting of paths in conda commands.
+- Fixed various issues with checkpoints.
+- Improved error messages when combining groups with cluster config.
+- Fixed bugs in group implementation.
+- Fixed singularity in combination with shadow. 
+
 [5.4.0] - 2018-12-18
 ====================
 

doc-environment.yml

+channels:
+  - conda-forge
+dependencies:
+  - python >=3.5
+  - datrie
+  - wrapt
+  - pyyaml
+  - requests
+  - appdirs
+  - docutils
+  - ratelimiter
+  - configargparse
+  - jsonschema
+  - gitpython
+  - sphinx
+  - pip:
+    - sphinxcontrib-napoleon
+  - sphinx-argparse
+  - sphinx_rtd_theme
+  - docutils==0.12
+  - recommonmark
+  - commonmark

docs/api_reference/internal/modules.rst

+Internal API
+============
+
+These pages document the entire internal API of Snakemake.
+
+.. toctree::
+   :maxdepth: 4
+
+   snakemake

docs/api_reference/internal/snakemake.remote.rst

+snakemake.remote package
+========================
+
+Submodules
+----------
+
+snakemake.remote.EGA module
+---------------------------
+
+.. automodule:: snakemake.remote.EGA
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.FTP module
+---------------------------
+
+.. automodule:: snakemake.remote.FTP
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.GS module
+--------------------------
+
+.. automodule:: snakemake.remote.GS
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.HTTP module
+----------------------------
+
+.. automodule:: snakemake.remote.HTTP
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.NCBI module
+----------------------------
+
+.. automodule:: snakemake.remote.NCBI
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.S3 module
+--------------------------
+
+.. automodule:: snakemake.remote.S3
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.S3Mocked module
+--------------------------------
+
+.. automodule:: snakemake.remote.S3Mocked
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.SFTP module
+----------------------------
+
+.. automodule:: snakemake.remote.SFTP
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.XRootD module
+------------------------------
+
+.. automodule:: snakemake.remote.XRootD
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.dropbox module
+-------------------------------
+
+.. automodule:: snakemake.remote.dropbox
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.gfal module
+----------------------------
+
+.. automodule:: snakemake.remote.gfal
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.gridftp module
+-------------------------------
+
+.. automodule:: snakemake.remote.gridftp
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.iRODS module
+-----------------------------
+
+.. automodule:: snakemake.remote.iRODS
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.remote.webdav module
+------------------------------
+
+.. automodule:: snakemake.remote.webdav
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: snakemake.remote
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/api_reference/internal/snakemake.report.rst

+snakemake.report package
+========================
+
+Module contents
+---------------
+
+.. automodule:: snakemake.report
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/api_reference/internal/snakemake.rst

+snakemake package
+=================
+
+Subpackages
+-----------
+
+.. toctree::
+
+    snakemake.remote
+    snakemake.report
+
+Submodules
+----------
+
+snakemake.benchmark module
+--------------------------
+
+.. automodule:: snakemake.benchmark
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.checkpoints module
+----------------------------
+
+.. automodule:: snakemake.checkpoints
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.common module
+-----------------------
+
+.. automodule:: snakemake.common
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.conda module
+----------------------
+
+.. automodule:: snakemake.conda
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.cwl module
+--------------------
+
+.. automodule:: snakemake.cwl
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.dag module
+--------------------
+
+.. automodule:: snakemake.dag
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.decorators module
+---------------------------
+
+.. automodule:: snakemake.decorators
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.exceptions module
+---------------------------
+
+.. automodule:: snakemake.exceptions
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.executors module
+--------------------------
+
+.. automodule:: snakemake.executors
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.gui module
+--------------------
+
+.. automodule:: snakemake.gui
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.io module
+-------------------
+
+.. automodule:: snakemake.io
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.jobs module
+---------------------
+
+.. automodule:: snakemake.jobs
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.logging module
+------------------------
+
+.. automodule:: snakemake.logging
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.output\_index module
+------------------------------
+
+.. automodule:: snakemake.output_index
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.parser module
+-----------------------
+
+.. automodule:: snakemake.parser
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.persistence module
+----------------------------
+
+.. automodule:: snakemake.persistence
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.rules module
+----------------------
+
+.. automodule:: snakemake.rules
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.scheduler module
+--------------------------
+
+.. automodule:: snakemake.scheduler
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.script module
+-----------------------
+
+.. automodule:: snakemake.script
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.shell module
+----------------------
+
+.. automodule:: snakemake.shell
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.singularity module
+----------------------------
+
+.. automodule:: snakemake.singularity
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.stats module
+----------------------
+
+.. automodule:: snakemake.stats
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.utils module
+----------------------
+
+.. automodule:: snakemake.utils
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.workflow module
+-------------------------
+
+.. automodule:: snakemake.workflow
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+snakemake.wrapper module
+------------------------
+
+.. automodule:: snakemake.wrapper
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: snakemake
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/getting_started/installation.rst

@@ -28,6 +28,18 @@ Then, you can install Snakemake with
     $ conda install -c bioconda -c conda-forge snakemake
 
 from the `Bioconda <https://bioconda.github.io>`_ channel.
+A minimal version of Snakemake which only depends on the bare necessities can be installed with
+
+.. code-block:: console
+
+    $ conda install -c bioconda -c conda-forge snakemake-minimal
+
+Note that Snakemake is available via Bioconda for historical, reproducibility, and continuity reasons.
+However, it is easy to combine Snakemake installation with other channels, e.g., by prefixing the package name with ``::bioconda``, i.e.,
+
+.. code-block:: console
+
+    $ conda install -c conda-forge bioconda::snakemake bioconda::snakemake-minimal
 
 Global Installation
 ===================

docs/index.rst

@@ -30,7 +30,6 @@ Workflows are described via a human readable, Python based language.
 They can be seamlessly scaled to server, cluster, grid and cloud environments, without the need to modify the workflow definition.
 Finally, Snakemake workflows can entail a description of required software, which will be automatically deployed to any execution environment.
 
-To get a first impression, see our `introductory slides <https://slides.com/johanneskoester/snakemake-short>`_.
 
 .. _manual-quick_example:
 
@@ -73,8 +72,9 @@ Rules describe how to create **output files** from **input files**.
 Getting started
 ---------------
 
+To get a first impression, see our `introductory slides <https://slides.com/johanneskoester/snakemake-short>`_ or watch the `live demo video <https://youtu.be/hPrXcUUp70Y>`_.
 News about Snakemake are published via `Twitter <https://twitter.com/search?l=&q=%23snakemake%20from%3Ajohanneskoester>`_.
-To get started, please do the :ref:`tutorial`, and see the :ref:`FAQ <project_info-faq>`.
+To learn Snakemake, please do the :ref:`tutorial`, and see the :ref:`FAQ <project_info-faq>`.
 
 .. _main-support:
 
@@ -139,6 +139,7 @@ Please consider to add your own.
 * Meyer et al. 2017. `Differential Gene Expression in the Human Brain Is Associated with Conserved, but Not Accelerated, Noncoding Sequences <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5400397/>`_. Molecular Biology and Evolution.
 * Lonardo et al. 2017. `Priming of soil organic matter: Chemical structure of added compounds is more important than the energy content <http://www.sciencedirect.com/science/article/pii/S0038071716304539>`_. Soil Biology and Biochemistry.
 * Beisser et al. 2017. `Comprehensive transcriptome analysis provides new insights into nutritional strategies and phylogenetic relationships of chrysophytes <https://peerj.com/articles/2832/>`_. PeerJ.
+* Piro et al 2017. `MetaMeta: integrating metagenome analysis tools to improve taxonomic profiling <https://microbiomejournal.biomedcentral.com/articles/10.1186/s40168-017-0318-y>`_. Microbiome.
 * Dimitrov et al 2017. `Successive DNA extractions improve characterization of soil microbial communities <https://peerj.com/articles/2915/>`_. PeerJ.
 * de Bourcy et al. 2016. `Phylogenetic analysis of the human antibody repertoire reveals quantitative signatures of immune senescence and aging <http://www.pnas.org/content/114/5/1105.short>`_. PNAS.
 * Bray et al. 2016. `Near-optimal probabilistic RNA-seq quantification <http://www.nature.com/nbt/journal/v34/n5/abs/nbt.3519.html>`_. Nature Biotechnology.
@@ -212,6 +213,7 @@ Please consider to add your own.
 
     api_reference/snakemake
     api_reference/snakemake_utils
+    api_reference/internal/modules
 
 
 .. toctree::

docs/project_info/contributing.rst

@@ -40,6 +40,101 @@ Implement Features
 Look through the Bitbucket issues for features.
 If you want to start working on an issue then please write short message on the issue tracker to prevent duplicate work.
 
+Contributing a new cluster or cloud execution backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Execution backends are added by implementing a so-called ``Executor``.
+All executors are located in `snakemake/executors.py <https://bitbucket.org/snakemake/snakemake/src/master/snakemake/executors.py>`_.
+In order to implement a new executor, you have to inherit from the class ``ClusterExecutor``.
+Below you find a skeleton
+
+.. code-block:: python
+
+    class SkeletonExecutor(ClusterExecutor):
+        def __init__(self, workflow, dag, cores,
+                 jobname="snakejob.{name}.{jobid}.sh",
+                 printreason=False,
+                 quiet=False,
+                 printshellcmds=False,
+                 latency_wait=3,
+                 cluster_config=None,
+                 local_input=None,
+                 restart_times=None,
+                 exec_job=None,
+                 assume_shared_fs=True,
+                 max_status_checks_per_second=1):
+
+            # overwrite the command to execute a single snakemake job if necessary
+            # exec_job = "..."
+
+            super().__init__(workflow, dag, None,
+                             jobname=jobname,
+                             printreason=printreason,
+                             quiet=quiet,
+                             printshellcmds=printshellcmds,
+                             latency_wait=latency_wait,
+                             cluster_config=cluster_config,
+                             local_input=local_input,
+                             restart_times=restart_times,
+                             exec_job=exec_job,
+                             assume_shared_fs=False,
+                             max_status_checks_per_second=10)
+
+            # add additional attributes
+
+        def shutdown(self):
+            # perform additional steps on shutdown if necessary
+            super().shutdown()
+
+        def cancel(self):
+            for job in self.active_jobs:
+                # cancel active jobs here
+            self.shutdown()
+
+        def run(self, job,
+                callback=None,
+                submit_callback=None,
+                error_callback=None):
+            import kubernetes.client
+
+            super()._run(job)
+            # obtain job execution command
+            exec_job = self.format_job(
+                self.exec_job, job, _quote_all=True,
+                use_threads="--force-use-threads" if not job.is_group() else "")
+
+            # submit job here, and obtain job ids from the backend
+
+            # register job as active, using your own namedtuple.
+            # The namedtuple must at least contain the attributes
+            # job, jobid, callback, error_callback.
+            self.active_jobs.append(MyJob(
+                job, jobid, callback, error_callback))
+
+        def _wait_for_jobs(self):
+            # busy wait on job completion
+            # This is only needed if your backend does not allow to use callbacks
+            # for obtaining job status.
+            while True:
+                # always use self.lock to avoid race conditions
+                with self.lock:
+                    if not self.wait:
+                        return
+                    active_jobs = self.active_jobs
+                    self.active_jobs = list()
+                    still_running = list()
+                for j in active_jobs:
+                    # use self.status_rate_limiter to avoid too many API calls.
+                    with self.status_rate_limiter:
+
+                        # Retrieve status of job j from your backend via j.jobid
+                        # Handle completion and errors, calling either j.callback(j.job)
+                        # or j.error_callback(j.job)
+                        # In case of error, add job j to still_running.
+                with self.lock:
+                    self.active_jobs.extend(still_running)
+                sleep()
+
 
 Write Documentation
 ===================

docs/requirements.txt

@@ -4,3 +4,5 @@ sphinx-argparse
 sphinx_rtd_theme
 docutils==0.12
 recommonmark
+configargparse
+appdirs

docs/snakefiles/rules.rst

@@ -426,6 +426,24 @@ Apart from Python scripts, this mechanism also allows you to integrate R_ and R
 
 In the R script, an S4 object named ``snakemake`` analog to the Python case above is available and allows access to input and output files and other parameters. Here the syntax follows that of S4 classes with attributes that are R lists, e.g. we can access the first input file with ``snakemake@input[[1]]`` (note that the first file does not have index ``0`` here, because R starts counting from ``1``). Named input and output files can be accessed in the same way, by just providing the name instead of an index, e.g. ``snakemake@input[["myfile"]]``.
 
+Finally, it is possible to integrate Julia_ scripts, e.g.
+
+.. _Julia: https://julialang.org
+
+.. code-block:: python
+
+    rule NAME:
+        input:
+            "path/to/inputfile",
+            "path/to/other/inputfile"
+        output:
+            "path/to/outputfile",
+            "path/to/another/outputfile"
+        script:
+            "path/to/script.jl"
+
+In the Julia_ script, a ``snakemake`` object is available, which can be accessed similar to the Python case (see above), with the only difference that you have to index from 1 instead of 0.
+
 For technical reasons, scripts are executed in ``.snakemake/scripts``. The original script directory is available as ``scriptdir`` in the ``snakemake`` object. A convenience method, ``snakemake@source()``, acts as a wrapper for the normal R ``source()`` function, and can be used to source files relative to the original script directory.
 
 An example external Python script would could look like this:

docs/tutorial/additional_features.rst

@@ -236,6 +236,40 @@ The **DRMAA support** can be activated by invoking Snakemake as follows:
 If available, **DRMAA is preferable over the generic cluster modes** because it provides better control and error handling.
 To support additional cluster specific parametrization, a Snakefile can be complemented by a :ref:`snakefiles-cluster_configuration` file.
 
+Using --cluster-status
+::::::::::::::::::::::
+
+Sometimes you need specific detection to determine if a cluster job completed successfully, failed or is still running.
+Error detection with ``--cluster`` can be improved for edge cases such as timeouts and jobs exceeding memory that are silently terminated by 
+the queueing system.
+This can be achieved with the ``--cluster-status`` option. This takes as input a script and passes a job id as first argument.
+
+The following (simplified) script detects the job status on a given SLURM cluster (>= 14.03.0rc1 is required for ``--parsable``).
+
+.. code:: python
+
+    #!/usr/bin/env python
+    import subprocess
+    import sys
+
+    jobid = sys.argv[1]
+
+    output = str(subprocess.check_output("sacct -j %s --format State --noheader | head -1 | awk '{print $1}'" % jobid, shell=True).strip())
+
+    running_status=["PENDING", "CONFIGURING", "COMPLETING", "RUNNING", "SUSPENDED"]
+    if "COMPLETED" in output:
+      print("success")
+    elif any(r in output for r in running_status):
+      print("running")
+    else:
+      print("failed")
+
+To use this script call snakemake similar to below, where ``status.py`` is the script above.
+
+.. code:: console
+
+    $ snakemake all --cluster "sbatch --cpus-per-task=1 --parsable" --cluster-status ./status.py
+
 
 Constraining wildcards
 ::::::::::::::::::::::
@@ -244,7 +278,7 @@ Snakemake uses regular expressions to match output files to input files and dete
 Sometimes it is useful to constrain the values a wildcard can have.
 This can be achieved by adding a regular expression that describes the set of allowed wildcard values.
 For example, the wildcard ``sample`` in the output file ``"sorted_reads/{sample}.bam"`` can be constrained to only allow alphanumeric sample names as ``"sorted_reads/{sample,[A-Za-z0-9]+}.bam"``.
-Constrains may be defined per rule or globally using the ``wildcard_constraints`` keyword, as demonstrated in :ref:`snakefiles-wildcards`.
+Constraints may be defined per rule or globally using the ``wildcard_constraints`` keyword, as demonstrated in :ref:`snakefiles-wildcards`.
 This mechanism helps to solve two kinds of ambiguity.
 
 * It can help to avoid ambiguous rules, i.e. two or more rules that can be applied to generate the same output file. Other ways of handling ambiguous rules are described in the Section :ref:`snakefiles-ambiguous-rules`.

docs/tutorial/advanced.rst

@@ -280,7 +280,7 @@ We use this mechanism for the output file of the rule ``bwa_map``:
 
 This results in the deletion of the BAM file once the corresponding ``samtools_sort`` job has been executed.
 Since the creation of BAM files via read mapping and sorting is computationally expensive, it is reasonable to **protect** the final BAM file **from accidental deletion or modification**.
-We modify the rule ``samtools_sort`` by marking it's output file as ``protected``:
+We modify the rule ``samtools_sort`` by marking its output file as ``protected``:
 
 .. code:: python
 

docs/tutorial/basics.rst

@@ -478,7 +478,7 @@ In total, the resulting workflow looks like this:
 
     rule all:
         input:
-            "report.html"
+            "plots/quals.svg"
 
 
     rule bwa_map:

setup.py

 # -*- coding: UTF-8 -*-
 
+from __future__ import print_function
+
 __author__ = "Johannes Köster"
 __copyright__ = "Copyright 2015, Johannes Köster"
 __email__ = "koester@jimmy.harvard.edu"

snakemake/__init__.py

@@ -26,6 +26,9 @@ from snakemake.utils import update_config, available_cpu_count
 from snakemake.common import Mode, __version__
 
 
+SNAKEFILE_CHOICES = ["Snakefile", "snakefile", "workflow/Snakefile", "workflow/snakefile"]
+
+
 def snakemake(snakefile,
               report=None,
               listrules=False,
@@ -214,7 +217,7 @@ def snakemake(snakefile,
         mode (snakemake.common.Mode): execution mode
         wrapper_prefix (str):       prefix for wrapper script URLs (default None)
         kubernetes (str):           submit jobs to kubernetes, using the given namespace.
-        kubernetes_env (list):      environment variables that shall be passed to kubernetes jobs.
+        kubernetes_envvars (list):  environment variables that shall be passed to kubernetes jobs.
         container_image (str):      Docker image to use, e.g., for kubernetes.
         default_remote_provider (str): default remote provider to use instead of local files (e.g. S3, GS)
         default_remote_prefix (str): prefix for default remote provider (e.g. name of the bucket).
@@ -323,7 +326,7 @@ def snakemake(snakefile,
             return False
 
     if not os.path.exists(snakefile):
-        logger.error("Error: Snakefile \"{}\" not present.".format(snakefile))
+        logger.error("Error: Snakefile \"{}\" not found.".format(snakefile))
         return False
     snakefile = os.path.abspath(snakefile)
 
@@ -367,7 +370,7 @@ def snakemake(snakefile,
             except ImportError as e:
                 raise WorkflowError("Unknown default remote provider.")
             if rmt.RemoteProvider.supports_default:
-                _default_remote_provider = rmt.RemoteProvider()
+                _default_remote_provider = rmt.RemoteProvider(keep_local=True, is_default=True)
             else:
                 raise WorkflowError("Remote provider {} does not (yet) support to "
                                     "be used as default provider.")
@@ -720,8 +723,15 @@ def get_argument_parser(profile=None):
 
     group_exec.add_argument("--snakefile", "-s",
                         metavar="FILE",
-                        default="Snakefile",
-                        help="The workflow definition in a snakefile.")
+                        help=("The workflow definition in form of a snakefile."
+                             "Usually, you should not need to specify this. "
+                             "By default, Snakemake will search for {} "
+                             "beneath the current working "
+                             "directory, in this order. "
+                             "Only if you definitely want a different layout, "
+                             "you need to use this parameter.").format(
+                                ", ".join(map("'{}'".format, SNAKEFILE_CHOICES))
+                            ))
     group_exec.add_argument(
         "--cores", "--jobs", "-j",
         action="store",
@@ -838,8 +848,11 @@ def get_argument_parser(profile=None):
     group_utils = parser.add_argument_group("UTILITIES")
 
     group_utils.add_argument("--report",
+                        nargs="?",
+                        const="report.html",
                         metavar="HTMLFILE",
-                        help="Create an HTML report with results and statistics.")
+                        help="Create an HTML report with results and statistics. "
+                             "If no filename is given, report.html is the default.")
     group_utils.add_argument("--export-cwl",
                         action="store",
                         metavar="FILE",
@@ -957,16 +970,15 @@ def get_argument_parser(profile=None):
         action="store_true",
         help="Remove all files generated by the workflow. Use together with --dryrun "
         "to list files without actually deleting anything. Note that this will "
-        "not recurse into subworkflows. It will also remove files flagged as "
-        "protected. Use with care!"
+        "not recurse into subworkflows. Write-protected files are not removed. "
+        "Nevertheless, use with care!"
     )
     group_utils.add_argument(
         "--delete-temp-output",
         action="store_true",
         help="Remove all temporary files generated by the workflow. Use together "
         "with --dryrun to list files without actually deleting anything. Note "
-        "that this will not recurse into subworkflows. It will also remove files "
-        "flagged as protected. Use with care!"
+        "that this will not recurse into subworkflows."
     )
     group_utils.add_argument(
         "--bash-completion",
@@ -1420,10 +1432,30 @@ def main(argv=None):
               "is set.", file=sys.stderr)
         sys.exit(1)
 
+    if args.kubernetes and (not args.default_remote_provider or
+                            not args.default_remote_prefix):
+        print("Error: --kubernetes must be combined with "
+              "--default-remote-provider and --default-remote-prefix, see "
+              "https://snakemake.readthedocs.io/en/stable/executable.html"
+              "#executing-a-snakemake-workflow-via-kubernetes", file=sys.stderr)
+        sys.exit(1)
+
     if args.delete_all_output and args.delete_temp_output:
         print("Error: --delete-all-output and --delete-temp-output are mutually exclusive.", file=sys.stderr)
         sys.exit(1)
 
+    if args.snakefile is None:
+        for p in SNAKEFILE_CHOICES:
+            if os.path.exists(p):
+                args.snakefile = p
+                break
+        if args.snakefile is None:
+            print("Error: no Snakefile found, tried {}.".format(
+                ", ".join(SNAKEFILE_CHOICES),
+                file=sys.stderr
+            ))
+            sys.exit(1)
+
     if args.gui is not None:
         try:
             import snakemake.gui as gui

snakemake/_version.py

@@ -23,9 +23,9 @@ def get_keywords():
     # setup.py/versioneer.py will grep for the variable names, so they must
     # each be defined on a line of their own. _version.py will just call
     # get_keywords().
-    git_refnames = " (HEAD -> master, tag: v5.4.0)"
-    git_full = "b2d791223686295c9539b37967978234e2f45897"
-    git_date = "2018-12-18 12:38:46 +0000"
+    git_refnames = " (HEAD -> master, tag: v5.5.3)"
+    git_full = "66c61c139e1f439e3f78c09df97927ad54363cc3"
+    git_date = "2019-07-11 18:27:18 +0200"
     keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
     return keywords
 

snakemake/common.py

@@ -20,6 +20,17 @@ SNAKEMAKE_SEARCHPATH = os.path.dirname(os.path.dirname(__file__))
 UUID_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "https://snakemake.readthedocs.io")
 
 
+def num_if_possible(s):
+    """Convert string to number if possible, otherwise return string."""
+    try:
+        return int(s)
+    except ValueError:
+        try:
+            return float(s)
+        except ValueError:
+            return s
+
+
 def get_last_stable_version():
     return __version__.split("+")[0]