.circleci/common.sh

-SINGULARITY_VER=3.3.0

.circleci/config.yml

-version: 2
-jobs:
-  build:
-    machine: true
-    environment:
-      - GCLOUD: /opt/google-cloud-sdk/bin/gcloud
-      - GSUTIL: /opt/google-cloud-sdk/bin/gsutil
-      - BOTO_CONFIG: /dev/null
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - snakemake-{{ .Environment.CACHE_VERSION }}-{{ checksum ".circleci/setup.sh" }}-{{ checksum "test-environment.yml" }}-{{ checksum ".circleci/common.sh" }}
-      - run:
-          name: Update PATH
-          command: echo 'export PATH="`pwd`/miniconda/bin:$PATH"' >> $BASH_ENV
-      - run:
-          name: Setup Conda
-          command: .circleci/setup.sh
-      - save_cache:
-          key: snakemake-{{ .Environment.CACHE_VERSION }}-{{ checksum ".circleci/setup.sh" }}-{{ checksum "test-environment.yml" }}-{{ checksum ".circleci/common.sh" }}
-          paths:
-            - miniconda
-      - run:
-          name: Setup singularity
-          command: |
-            # TODO only install if singularity is not yet present
-            # if type singularity > /dev/null; then exit 0; fi
-            source .circleci/common.sh
-            sudo add-apt-repository ppa:gophers/archive
-            sudo apt-get update
-            sudo apt-get install build-essential libssl-dev uuid-dev libgpgme11-dev libseccomp-dev wget pkg-config squashfs-tools libarchive-dev golang-1.11
-            export PATH=/usr/lib/go-1.11/bin:$PATH
-            wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VER}/singularity-${SINGULARITY_VER}.tar.gz
-            tar -xvf singularity-$SINGULARITY_VER.tar.gz
-            cd singularity
-            ./mconfig
-            make -C builddir
-            sudo make -C builddir install
-      - run:
-          name: Setup Snakemake
-          command: |
-            source activate snakemake
-            pip install -e .
-      - run:
-          name: Setup iRODS Docker image
-          command: |
-            docker build -t irods-server tests/test_remote_irods
-            docker run -d -p 1247:1247 --name provider irods-server -i run_irods
-            sleep 10
-            docker exec -u irods provider iput /incoming/infile
-            cp -r tests/test_remote_irods/setup-data ~/.irods
-      - run:
-          name: Setup gcloud
-          command: |
-            # skip if key is unset
-            if [ -z $GCLOUD_SERVICE_KEY ]; then exit 0; fi
-            # otherwise init cloud
-            echo $GCLOUD_SERVICE_KEY | base64 --decode --ignore-garbage > ${HOME}/gcloud-service-key.json
-            sudo $GCLOUD components install kubectl
-            sudo chown -R $USER:$USER /home/circleci/.config/gcloud
-            $GCLOUD auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
-            $GCLOUD config set project snakemake-testing
-      - run:
-          name: Run tests
-          no_output_timeout: 60m
-          command: |
-            export GCLOUD_CLUSTER=t-`uuidgen`
-            export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcloud-service-key.json
-            source activate snakemake
-            py.test tests/test*.py -v -x

.circleci/setup.sh

-#!/bin/bash
-set -euo pipefail
-
-if type conda > /dev/null; then exit 0; fi
-wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-bash miniconda.sh -b -p miniconda
-conda env create --name snakemake --file test-environment.yml

.coveragerc

+[run]
+source = snakemake
+parallel = True
+
+[report]
+omit = tests/*

.gitattributes

 snakemake/_version.py export-subst
+snakemake/report/report.html linguist-vendored=true
+snakemake/gui.html linguist-vendored=true
+snakemake/report/table.html linguist-vendored=true
+tests/test_issue1046/expected-results/test_report.html linguist-vendored=true

.github/ISSUE_TEMPLATE/bug_report.md

+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+<!-- Please do not post usage questions here. Ask them on Stack Overflow: https://stackoverflow.com/questions/tagged/snakemake -->
+
+**Snakemake version**
+Note the Snakemake version for which you experience the bug.
+Always make sure to try out the latest version before creating a new bug report.
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Logs**
+If applicable, any terminal output to help explain your problem.
+
+**Minimal example**
+Add a minimal example for reproducing the bug.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/main.yml

+name: CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches_ignore: []
+
+jobs:
+  formatting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup black environment
+        run: conda create --quiet --name black black
+
+      - name: Check formatting
+        run: |
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate black
+          black --check snakemake tests/*.py
+
+  testing:
+    runs-on: ubuntu-latest
+    needs: formatting
+    steps:
+      - uses: actions/checkout@v1
+      
+      - name: Setup Snakemake environment
+        run: conda env create --quiet --name snakemake --file test-environment.yml
+
+      - name: Setup apt dependencies
+        run: sudo apt install -y singularity-container stress
+        
+      - name: Setup iRODS
+        run: |
+          docker build -t irods-server tests/test_remote_irods
+          docker run -d -p 1247:1247 --name provider irods-server -i run_irods
+          sleep 10
+          docker exec -u irods provider iput /incoming/infile
+          cp -r tests/test_remote_irods/setup-data ~/.irods
+
+      - name: Run tests
+        env:
+          CI: true
+        run: |
+          # enable coverage recording for subprocesses
+                echo -e "try:\n    import coverage\n    coverage.process_startup()\nexcept:\n    pass" > sitecustomize.py
+          export COVERAGE_PROCESS_START=.coveragerc
+
+          # activate conda env
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate snakemake
+          
+          # run tests
+          coverage run -m pytest tests/test*.py -v -x
+
+          # collect coverage report
+          coverage combine
+          coverage xml
+
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v1.0.3
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}
+
+      - name: Build container image
+        run: docker build .

.gitignore

@@ -9,6 +9,9 @@ dist/
 *.egg
 .eggs/
 .snakemake*
+.venv
+.venv/*
+
 
 .idea
 

CHANGELOG.rst

+[5.7.4] - 2019-10-23
+====================
+Changed
+-------
+- Various fixes and adaptations in the docker container image and the test suite.
+
+[5.7.1] - 2019-10-16
+====================
+Added
+-----
+- Ability to print log files of failed jobs with --show-failed-logs.
+Changed
+-------
+- Fixed bugs in tibanna executor.
+- Fixed handling of symbolic links.
+- Fixed typos in help texts.
+- Fixed handling of default resources.
+- Fixed bugs in azure storage backend.
+
+[5.7.0] - 2019-10-07
+====================
+Changed
+-------
+- Fixed various corner case bugs. Many thanks to the community for pull requests and reporting!
+- Container execution adapted to latest singularity.
+
+Added
+-----
+- First class support for Amazon cloud execution via a new `Tibanna backend <https://snakemake.readthedocs.io/en/v5.7.0/executable.html#executing-a-snakemake-workflow-via-tibanna-on-amazon-web-services>`. Thanks to Soo Lee from Harvard Biomedical Informatics!
+- Allow multiple config files to be passed via the command line.
+- A new, more detailed way to visualize the DAG (--filegraph). Thanks to Henning Timm!
+- Pathlib compatibility added. Input and output files can now also be Path objects. Thanks to Frederik Boulund!
+- New azure storage remote provider. Transparently access input and output files on Microsoft Azure. Thanks to Sebastian Kurscheid!
+
 [5.6.0] - 2019-09-06
 ====================
 Changed

Dockerfile

 FROM bitnami/minideb:stretch
 MAINTAINER Johannes Köster <johannes.koester@tu-dortmund.de>
-ENV SINGULARITY_VERSION=2.4.5
 ADD . /tmp/repo
 WORKDIR /tmp/repo
 ENV PATH /opt/conda/bin:${PATH}
 ENV LANG C.UTF-8
 ENV SHELL /bin/bash
-RUN install_packages wget bzip2 ca-certificates gnupg2 squashfs-tools git && \
+RUN /bin/bash -c "install_packages wget bzip2 ca-certificates gnupg2 squashfs-tools git && \
     wget -O- http://neuro.debian.net/lists/xenial.us-ca.full > /etc/apt/sources.list.d/neurodebian.sources.list && \
     wget -O- http://neuro.debian.net/_static/neuro.debian.net.asc | apt-key add - && \
     install_packages singularity-container && \
     wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
     bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \
     rm Miniconda3-latest-Linux-x86_64.sh && \
-    conda update -n base conda && conda env update --name root --file /tmp/repo/environment.yml && conda clean --all -y && \
-    pip install .
+    conda env create -n snakemake --file /tmp/repo/environment.yml && \
+    conda clean --all -y && \
+    source activate snakemake && \
+    which python && \
+    pip install ."
+RUN echo "source activate snakemake" > ~/.bashrc
+ENV PATH /opt/conda/envs/snakemake/bin:${PATH}

MANIFEST.in

 include versioneer.py
 include snakemake/_version.py
+include LICENSE.md

README.md

-[![CircleCI](https://circleci.com/bb/snakemake/snakemake/tree/master.svg?style=shield)](https://circleci.com/bb/snakemake/snakemake/tree/master)
+[![CircleCI](https://circleci.com/gh/snakemake/snakemake/tree/master.svg?style=shield)](https://circleci.com/gh/snakemake/snakemake/tree/master)
+[![Sonarcloud Status](https://sonarcloud.io/api/project_badges/measure?project=snakemake_snakemake&metric=alert_status)](https://sonarcloud.io/dashboard?id=snakemake_snakemake)
+[![Bioconda](https://img.shields.io/conda/dn/bioconda/snakemake.svg?label=Bioconda)](https://bioconda.github.io/recipes/snakemake/README.html)
+[![Pypi](https://img.shields.io/pypi/pyversions/snakemake.svg)](https://pypi.org/project/snakemake)
+[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/snakemake/snakemake)](https://hub.docker.com/r/snakemake/snakemake)
+[![Stack Overflow](https://img.shields.io/badge/stack-overflow-orange.svg)](https://stackoverflow.com/questions/tagged/snakemake)
+[![Twitter](https://img.shields.io/twitter/follow/johanneskoester.svg?style=social&label=Follow)](https://twitter.com/search?l=&q=%23snakemake%20from%3Ajohanneskoester)
+[![Github stars](https://img.shields.io/github/stars/snakemake/snakemake?style=social)](https://github.com/snakemake/snakemake/stargazers)
 
-# Snakemake - a pythonic workflow system
+# Snakemake
 
-Snakemake is a workflow management system that aims to reduce the complexity of creating workflows 
-by providing a fast and comfortable execution environment, together with a clean and readable 
-specification language in Python style. Snakemake workflows are essentially Python scripts extended 
-by declarative code to define rules. Rules describe how to create output files from input files.
+The Snakemake workflow management system is a tool to create **reproducible and scalable** data analyses.
+Workflows are described via a human readable, Python based language.
+They can be seamlessly scaled to server, cluster, grid and cloud environments, without the need to modify the workflow definition.
+Finally, Snakemake workflows can entail a description of required software, which will be automatically deployed to any execution environment.
 
-Homepage: https://snakemake.bitbucket.io
+**Homepage: https://snakemake.readthedocs.io**
 
-Copyright (c) 2012-2017 Johannes Köster <johannes.koester@protonmail.com> (see LICENSE)
+**Note: Snakemake has just (Oct 4 2019) moved from bitbucket.org/snakemake/snakemake to Github. Forks and stars are not yet insightful here!**
+
+Copyright (c) 2012-2019 Johannes Köster <johannes.koester@uni-due.com> (see LICENSE)

debian/changelog

+snakemake (5.7.4-1) unstable; urgency=medium
+
+  * Team upload.
+  * New upstream release.
+  * Skip test_url_include in the autopkgtest
+  * debian/{control,copyight,watch}: upstream is now using GitHub
+  * Set upstream metadata fields: Repository.
+  * Adapt some of the new tests to use python3 instead of python.
+
+ -- Michael R. Crusoe <michael.crusoe@gmail.com>  Wed, 30 Oct 2019 17:22:29 +0100
+
 snakemake (5.6.0-2) unstable; urgency=medium
 
   * Team upload.

debian/clean

 fakehome/
+.pytest_cache/

debian/control

@@ -9,6 +9,7 @@ Build-Depends: debhelper-compat (= 12),
                cwltool,
                imagemagick,
                python3,
+               python3-biopython,
                python3-boto,
                python3-botocore,
                python3-configargparse,
@@ -30,16 +31,18 @@ Build-Depends: debhelper-compat (= 12),
                python3-rpy2,
                python3-setuptools,
                python3-sphinx,
+               python3-sphinx-argparse,
                python3-sphinx-bootstrap-theme,
                python3-sphinx-rtd-theme,
                python3-wrapt,
                python3-yaml,
-               r-cran-rmarkdown
+               r-cran-rmarkdown,
+               stress
 # python3-irodsclient,  # when that enters testing
 Standards-Version: 4.4.0
 Vcs-Browser: https://salsa.debian.org/med-team/snakemake
 Vcs-Git: https://salsa.debian.org/med-team/snakemake.git
-Homepage: https://bitbucket.org/snakemake/snakemake
+Homepage: https://snakemake.readthedocs.io/
 
 Package: snakemake
 Architecture: all
@@ -62,6 +65,7 @@ Depends: ${misc:Depends},
          python3-wrapt,
          python3-yaml
 Recommends: cwltool,
+            python3-biopython,
             python3-boto,
             python3-botocore,
             python3-dropbox,

debian/copyright

 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: snakemake
 Upstream-Contact: Johannes Köster <johannes.koester@tu-dortmund.de>
-Source: https://bitbucket.org/snakemake/snakemake
+Source: https://github.com/snakemake/snakemake
 Files-Excluded: docker
 
 Files: *

debian/patches/0003-Use-debian-s-mathjax-package.patch

@@ -8,8 +8,8 @@ Use debian's mathjax package
  snakemake/gui.html | 6 +++---
  2 files changed, 6 insertions(+), 3 deletions(-)
 
---- a/docs/conf.py
-+++ b/docs/conf.py
+--- snakemake.orig/docs/conf.py
++++ snakemake/docs/conf.py
 @@ -41,6 +41,9 @@
      'sphinxarg.ext'
  ]
@@ -20,11 +20,12 @@ Use debian's mathjax package
  # Add any paths that contain templates here, relative to this directory.
  templates_path = ['_templates']
  
---- a/snakemake/gui.html
-+++ b/snakemake/gui.html
-@@ -1,11 +1,11 @@
- <html>
+--- snakemake.orig/snakemake/gui.html
++++ snakemake/snakemake/gui.html
+@@ -2,12 +2,12 @@
+ <html lang="en">
      <head>
+         <title>Snakemake</title>
 -        <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
 +        <script type="text/javascript" src="file:///usr/share/javascript/jquery/jquery.min.js"></script>
          <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/d3/3.4.6/d3.min.js"></script>

debian/patches/0011-fix-privacy-breach.patch

@@ -9,19 +9,19 @@ Description: Avoid privacy breach
  This patch does not solve all issues.  The remaining ones are discussed
  in debian/README.source.
 
---- a/snakemake/gui.html
-+++ b/snakemake/gui.html
-@@ -1,7 +1,7 @@
- <html>
+--- snakemake.orig/snakemake/gui.html
++++ snakemake/snakemake/gui.html
+@@ -3,7 +3,7 @@
      <head>
+         <title>Snakemake</title>
          <script type="text/javascript" src="file:///usr/share/javascript/jquery/jquery.min.js"></script>
 -        <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/d3/3.4.6/d3.min.js"></script>
 +        <script type="text/javascript" src="file:///usr/share/javascript/d3/d3.min.js"></script>
          <script type="text/javascript" src="file:///usr/share/javascript/bootstrap/js/bootstrap.min.js"></script>
          <script type="text/javascript" src="http://cpettitt.github.io/project/dagre-d3/v0.1.5/dagre-d3.js"></script>
          <script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/bootstrap-select/1.5.4/bootstrap-select.min.js"></script>
---- a/docs/index.rst
-+++ b/docs/index.rst
+--- snakemake.orig/docs/index.rst
++++ snakemake/docs/index.rst
 @@ -4,25 +4,25 @@
  Snakemake
  =========
@@ -38,31 +38,30 @@ Description: Avoid privacy breach
 +.. image:: file:///usr/share/doc/snakemake/html/_svg/snakemake.svg
      :target: https://pypi.python.org/pypi/snakemake
  
--.. image:: https://quay.io/repository/snakemake/snakemake/status
-+.. image:: file:///usr/share/doc/snakemake/html/_svg/status.svg
-        :target: https://quay.io/repository/snakemake/snakemake
+ .. image:: https://img.shields.io/docker/cloud/build/snakemake/snakemake
+        :target: https://hub.docker.com/r/snakemake/snakemake
  
--.. image:: https://circleci.com/bb/snakemake/snakemake/tree/master.svg?style=shield
+-.. image:: https://circleci.com/gh/snakemake/snakemake/tree/master.svg?style=shield
 +.. image:: file:///usr/share/doc/snakemake/html/_svg/master.svg
-     :target: https://circleci.com/bb/snakemake/snakemake/tree/master
+     :target: https://circleci.com/gh/snakemake/snakemake/tree/master
  
 -.. image:: https://img.shields.io/badge/stack-overflow-orange.svg
 +.. image:: file:///usr/share/doc/snakemake/html/_svg/stack-overflow-orange.svg
-     :target: http://stackoverflow.com/questions/tagged/snakemake
+     :target: https://stackoverflow.com/questions/tagged/snakemake
  
 -.. image:: https://img.shields.io/twitter/follow/johanneskoester.svg?style=social&label=Follow
 +.. image:: file:///usr/share/doc/snakemake/html/_svg/johanneskoester_follow.svg
      :target: https://twitter.com/search?l=&q=%23snakemake%20from%3Ajohanneskoester
  
- The Snakemake workflow management system is a tool to create **reproducible and scalable** data analyses.
---- a/docs/project_info/citations.rst
-+++ b/docs/project_info/citations.rst
+ .. image:: https://img.shields.io/github/stars/snakemake/snakemake?style=social
+--- snakemake.orig/docs/project_info/citations.rst
++++ snakemake/docs/project_info/citations.rst
 @@ -38,7 +38,7 @@
  
  If you publish a Snakemake workflow, consider to add this badge to your project page:
  
--.. image:: https://img.shields.io/badge/snakemake-≥3.5.2-brightgreen.svg?style=flat-square
+-.. image:: https://img.shields.io/badge/snakemake-≥5.6.0-brightgreen.svg?style=flat
 +.. image:: file:///usr/share/doc/snakemake/html/_svg/snakemake_gt_3.5.2-brightgreen.svg
-    :target: https://snakemake.bitbucket.io
+    :target: https://snakemake.readthedocs.io
  
  The markdown syntax is