.gitignore

@@ -10,7 +10,8 @@ __pycache__
 .idea/
 /test-report.xml
 /test-report-*.xml
-/venv/
+venv/
+v3nv/
 tmp/
 /src/toil/test/cwl/spec
 /cwltool_deps/
@@ -18,3 +19,4 @@ tmp/
 /docker/Dockerfile
 /docker/toil-*.tar.gz
 /src/toil/version.py
+.coverage*

.gitlab-ci.yml

+before_script:
+  - docker info
+  - cat /etc/hosts
+after_script:
+  # We need to clean up any files that Toil may have made via Docker that
+  # aren't deletable by the Gitlab user. If we don't do this, Gitlab will try
+  # and clean them up before running the next job on the runner, fail, and fail
+  # that next job.
+  - pwd
+  - sudo rm -rf tmp
+
+
+stages:
+  - test
+
+
+py2-job:
+  stage: test
+  script:
+    - pwd
+    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor && PYTHONIOENCODING=utf-8 make test
+
+py3-job:
+  stage: test
+  script:
+    - pwd
+    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor && PYTHONIOENCODING=utf-8 make test

.travis.yml

 language: python
 python:
+  - "3.5"
+  - "3.6"
   - "2.7"
 install:
   - make prepare
   - make develop extras=[aws,google] # adding extras to avoid import errors
 script:
   - TOIL_TEST_QUICK=True make test_offline
+env:
+  # Necessary to get boto to work in Travis's Ubuntu Precise
+  # environment (see #2498). Consider removing this if/when we
+  # transition to the Xenial environment.
+  - BOTO_CONFIG=/dev/null

Makefile

@@ -90,8 +90,8 @@ help:
 
 # This Makefile uses bash features like printf and <()
 SHELL=bash
-python=python2.7
-pip=pip2.7
+python=python
+pip=pip
 tests=src
 tests_local=src/toil/test
 # do slightly less than travis timeout of 10 min.
@@ -267,8 +267,7 @@ check_build_reqs:
 
 
 prepare: check_venv
-	$(pip) install sphinx==1.5.5 mock==1.0.1 pytest==3.6.2 stubserver==1.0.1 \
-		pytest-timeout==1.2.0 cwltest
+	$(pip) install mock==1.0.1 pytest==4.3.1 pytest-cov==2.6.1 stubserver==1.0.1 pytest-timeout==1.3.3 cwltest
 
 
 check_venv:

README.rst

 ATTENTION: Toil has moved from https://github.com/BD2KGenomics/toil to https://github.com/DataBiosphere/toil as of July 5th, 2018.
 
-Toil is a scalable, efficient, cross-platform pipeline management system,
+Toil is a scalable, efficient, cross-platform (Linux & macOS) pipeline management system,
 written entirely in Python, and designed around the principles of functional
 programming.
 
+Our next scheduled release is May 24, 2019.
+
 * Check the `website`_ for a description of Toil and its features.
 * Full documentation for the latest stable release can be found at
   `Read the Docs`_.

contrib/admin/test-pr

+#!/usr/bin/env bash
+
+# test-pr: submit a PR from an external contributor for Gitlab testing
+
+set -e
+
+# Check arguments
+if [[ "${#}" -ne 3 ]] ; then
+    printf >&2 "test-pr: submit a PR from an external contributor for Gitlab testing\n\n"
+    printf >&2 "\tusage: test-pr GH_USER SOURCE_BRANCH DEST_BRANCH\n\n"
+    printf >&2 "\texample: test-pr arostamianfar defaultcaching issues/2586-allow-enabling-caching\n\n"
+    exit 1
+fi
+
+# Parse arguments
+GH_USER="${1}"
+shift
+SOURCE_BRANCH="${1}"
+shift
+DEST_BRANCH="${1}"
+shift
+
+echo "Fetch ${SOURCE_BRANCH} from ${GH_USER}"
+git fetch "git@github.com:${GH_USER}/toil.git" "${SOURCE_BRANCH}"
+echo "Push to Toil ${DEST_BRANCH}"
+git push git@github.com:DataBiosphere/toil.git "FETCH_HEAD:refs/heads/${DEST_BRANCH}"

dashboard/prometheus/prometheus.yml

@@ -33,8 +33,8 @@
           - targets: ['localhost:9100']
       - job_name: 'aws-node-exporter'
         ec2_sd_configs:
-          - region: 'us-west-2'
-            refresh_interval: 15s
+          - refresh_interval: 15s
+            region: ZONE_NAME
             port: 9100
         relabel_configs:
           - source_labels: ['__meta_ec2_tag_Name']

dashboard/prometheus/run.sh

@@ -4,5 +4,6 @@
 #Set cluster name
 
 sed -i "s/^\([[:blank:]]*\) regex: CLUSTER_NAME/\1 regex: \'${1}\'/g" /etc/prometheus/prometheus.yml
+sed -i "s/^\([[:blank:]]*\) region: ZONE_NAME/\1 region: \'${2}\'/g" /etc/prometheus/prometheus.yml
 
 exec /bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles

debian/changelog

-toil (3.18.0-3) UNRELEASED; urgency=medium
+toil (3.20.0-1) UNRELEASED; urgency=medium
 
+  [ Michael R. Crusoe ]
   * Make the build reproducible, patch from lamby@debian.org; thanks!
     (Closes: #925191)
 
+  [ Andreas Tille ]
+  * debhelper-compat 12
+  * Point Vcs fields to salsa.debian.org
+  * Standards-Version: 4.4.0
+  TODO: Finallise quilt patches (please someone take over from here!)
+
  -- Michael R. Crusoe <michael.crusoe@gmail.com>  Wed, 20 Mar 2019 21:46:34 -0500
 
 toil (3.18.0-2) unstable; urgency=medium

debian/compat

-11

debian/control

 Source: toil
-Section: net
-Priority: optional
 Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
 Uploaders: Steffen Moeller <moeller@debian.org>,
            Michael R. Crusoe <michael.crusoe@gmail.com>
-Build-Depends: debhelper (>= 11~),
+Section: net
+Priority: optional
+Build-Depends: debhelper-compat (= 12),
                dh-python,
                python3-all,
                python3-setuptools,
@@ -17,23 +17,26 @@ Build-Depends: debhelper (>= 11~),
                python3-urllib3,
                python3-chardet,
                python3-sphinx
-Standards-Version: 4.3.0
-Homepage: https://github.com/BD2KGenomics/toil
-Vcs-Git: https://salsa.debian.org/med-team/toil.git
+Standards-Version: 4.4.0
 Vcs-Browser: https://salsa.debian.org/med-team/toil
+Vcs-Git: https://salsa.debian.org/med-team/toil.git
+Homepage: https://github.com/BD2KGenomics/toil
 
 Package: toil
 Architecture: all
-Provides: cwl-runner
-Depends: ${python3:Depends}, ${misc:Depends},
+Depends: ${python3:Depends},
+         ${misc:Depends},
          cwltool
-Recommends:
-         python3-boto, python3-boto3,
+Recommends: python3-boto,
+            python3-boto3,
 # for google cloud support
             python3-libcloud
 # for mesos support, but needs python3-mesos first
 #         python3-protobuf,
-Suggests: docker.io | singularity-container | docker-ce, python3-azure, python3-azure-storage
+Suggests: docker.io | singularity-container | docker-ce,
+          python3-azure,
+          python3-azure-storage
+Provides: cwl-runner
 #Suggests: python-toil-doc
 Description: cross-platform workflow engine
  Toil is a scalable, efficient, cross-platform and easy-to-use workflow

debian/patches/adjust_to_newer_cwltool

 Author: Michael R. Crusoe <michael.crusoe@gmail.com>
 Description: Adjust to newer version of cwltool
---- toil.orig/src/toil/cwl/cwltoil.py
-+++ toil/src/toil/cwl/cwltoil.py
-@@ -40,6 +40,7 @@
- 
+--- a/src/toil/cwl/cwltoil.py
++++ b/src/toil/cwl/cwltoil.py
+@@ -41,6 +41,7 @@ import six
  from schema_salad import validate
+ from schema_salad.schema import Names
  import schema_salad.ref_resolver
 +from schema_salad.schema import Names
  
  import cwltool.errors
  import cwltool.load_tool
-@@ -58,7 +59,7 @@
-                                 get_listing, MapperEnt, visit_class,
-                                 normalizeFilesDirs)
- from cwltool.process import (shortname, fill_in_defaults, compute_checksums,
--                             collectFilesAndDirs, add_sizes)
-+                             add_sizes)
- from cwltool.secrets import SecretStore
- from cwltool.software_requirements import (
-     DependenciesConfiguration, get_container_from_software_requirements)
-@@ -389,8 +390,21 @@
-     """Copy input files out of the global file store and update location and
-     path."""
- 
--    jobfiles = []  # type: List[Dict[Text, Any]]
--    collectFilesAndDirs(cwljob, jobfiles)
-+    def _collectDirEntries(obj):
-+        # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> Iterator[Dict[Text, Any]]
-+        if isinstance(obj, dict):
-+            if obj.get("class") in ("File", "Directory"):
-+                yield obj
-+            else:
-+                for sub_obj in obj.values():
-+                    for dir_entry in _collectDirEntries(sub_obj):
-+                        yield dir_entry
-+        elif isinstance(obj, list):
-+            for sub_obj in obj:
-+                for dir_entry in _collectDirEntries(sub_obj):
-+                    yield dir_entry
-+
-+    jobfiles = list(_collectDirEntries(cwljob))
-     pm = ToilPathMapper(
-         jobfiles, "", outdir, separateDirs=False, stage_listing=True)
-     for f, p in pm.items():
-@@ -455,6 +469,7 @@
+@@ -469,6 +470,7 @@ class CWLJobWrapper(Job):
          self.addChild(realjob)
          return realjob.rv()
  
@@ -51,53 +18,7 @@ Description: Adjust to newer version of cwltool
  def _makeNestedTempDir(top, seed, levels=2):
      """
      Gets a temporary directory in the hierarchy of directories under a given
-@@ -497,12 +512,29 @@
-         if runtime_context.builder:
-             builder = runtime_context.builder
-         else:
--            builder = cwltool.builder.Builder(cwljob)
--            builder.requirements = self.cwltool.requirements
--            builder.outdir = None
--            builder.tmpdir = None
--            builder.timeout = runtime_context.eval_timeout
--            builder.resources = {}
-+            builder = cwltool.builder.Builder(
-+                job=cwljob,
-+                files=[],
-+                bindings=[],
-+                schemaDefs={},
-+                names=Names(),
-+                requirements=self.cwltool.requirements,
-+                hints=[],
-+                resources={},
-+                mutation_manager=None,
-+                formatgraph=None,
-+                make_fs_access=runtime_context.make_fs_access,
-+                fs_access=runtime_context.make_fs_access(''),
-+                job_script_provider=None,
-+                timeout=runtime_context.eval_timeout,
-+                debug=False,
-+                js_console=False,
-+                force_docker_pull=False,
-+                loadListing=u'',
-+                outdir=u'',
-+                tmpdir=u'',
-+                stagedir=u''
-+            )
-         req = tool.evalResources(builder, runtime_context)
-         # pass the default of None if basecommand is empty
-         unitName = self.cwltool.tool.get("baseCommand", None)
-@@ -530,7 +562,8 @@
-         fill_in_defaults(
-             self.step_inputs, cwljob,
-             self.runtime_context.make_fs_access(""))
--        for inp_id in cwljob.keys():
-+        immobile_cwljob_dict = copy.deepcopy(cwljob)
-+        for inp_id in immobile_cwljob_dict.keys():
-             found = False
-             for field in self.cwltool.inputs_record_schema['fields']:
-                 if field['name'] == inp_id:
-@@ -576,6 +609,7 @@
+@@ -615,6 +617,7 @@ class CWLJob(Job):
  
          return output
  
@@ -105,7 +26,7 @@ Description: Adjust to newer version of cwltool
  def makeJob(tool, jobobj, step_inputs, runtime_context):
      """Create the correct Toil Job object for the CWL tool (workflow, job, or job
      wrapper for dynamic resource requirements.)
-@@ -1079,7 +1113,8 @@
+@@ -1149,7 +1152,8 @@ def main(args=None, stdout=sys.stdout):
  
      if options.provisioner and not options.jobStore:
          raise NoSuchJobStoreException(
@@ -115,18 +36,7 @@ Description: Adjust to newer version of cwltool
  
      use_container = not options.no_container
  
-@@ -1141,9 +1176,9 @@
-             document_loader, avsc_names, processobj, metadata, uri = \
-                 cwltool.load_tool.validate_document(
-                     document_loader, workflowobj, uri,
-+                    loading_context.overrides_list, loading_context.metadata,
-                     loading_context.enable_dev, loading_context.strict, False,
-                     loading_context.fetcher_constructor, False,
--                    loading_context.overrides_list,
-                     do_validate=loading_context.do_validate)
-             loading_context.overrides_list.extend(
-                 metadata.get("cwltool:overrides", []))
-@@ -1215,7 +1250,8 @@
+@@ -1271,7 +1275,8 @@ def main(args=None, stdout=sys.stdout):
                  return 33
  
              wf1.cwljob = initialized_job_order
@@ -136,22 +46,11 @@ Description: Adjust to newer version of cwltool
                  wf1.addFollowOnFn(cleanTempDirs, wf1)
              outobj = toil.start(wf1)
  
---- toil.orig/src/toil/common.py
-+++ toil/src/toil/common.py
-@@ -1083,7 +1083,7 @@
-         So far there is no reason to store any old pids.
-         """
-         with self._jobStore.writeSharedFileStream('pid.log') as f:
--            f.write(str(os.getpid()))
-+            f.write(str(os.getpid()).encode('utf-8'))
- 
- 
- class ToilRestartException(Exception):
---- toil.orig/src/toil/jobGraph.py
-+++ toil/src/toil/jobGraph.py
-@@ -100,6 +100,9 @@
-         # this job
-         self.chainedJobs = chainedJobs
+--- a/src/toil/jobGraph.py
++++ b/src/toil/jobGraph.py
+@@ -103,6 +103,9 @@ class JobGraph(JobNode):
+     def __hash__(self):
+         return hash(self.jobStoreID)
  
 +    def __hash__(self):
 +        return hash(self.jobStoreID)

debian/patches/debianize_docs

 Author: Michael R. Crusoe <michael.crusoe@gmail.com>
 Description: Update docs to reflect a local install
---- toil.orig/docs/index.rst
-+++ toil/docs/index.rst
-@@ -31,7 +31,6 @@
+--- a/docs/index.rst
++++ b/docs/index.rst
+@@ -31,7 +31,6 @@ If using Toil for your research, please
     :caption: Getting Started
     :maxdepth: 2
  
@@ -10,9 +10,9 @@ Description: Update docs to reflect a local install
     gettingStarted/quickStart
  
  .. toctree::
---- toil.orig/docs/gettingStarted/quickStart.rst
-+++ toil/docs/gettingStarted/quickStart.rst
-@@ -8,11 +8,9 @@
+--- a/docs/gettingStarted/quickStart.rst
++++ b/docs/gettingStarted/quickStart.rst
+@@ -8,22 +8,15 @@ Quickstart Examples
  Running a basic workflow
  ------------------------
  
@@ -24,11 +24,7 @@ Description: Update docs to reflect a local install
 -2. Copy and paste the following code block into a new file called ``helloWorld.py``:
 +1. Copy and paste the following code block into a new file called ``helloWorld.py``:
  
-    .. code-block:: python
- 
-@@ -30,14 +28,9 @@
-           print output
- 
+ .. literalinclude:: ../../src/toil/test/docs/scripts/tutorial_helloworld.py
  
 -3. Specify the name of the :ref:`job store <jobStoreOverview>` and run the workflow::
 -
@@ -43,7 +39,7 @@ Description: Update docs to reflect a local install
  
  Congratulations! You've run your first Toil workflow using the default :ref:`Batch System <batchsysteminterface>`, ``singleMachine``,
  using the ``file`` job store.
-@@ -64,13 +57,6 @@
+@@ -50,13 +43,6 @@ The `Common Workflow Language`_ (CWL) is
  workflows that are portable across multiple workflow engines and platforms.
  Running CWL workflows using Toil is easy.
  
@@ -57,7 +53,7 @@ Description: Update docs to reflect a local install
  #. Copy and paste the following code block into ``example.cwl``:
  
     .. code-block:: yaml
-@@ -96,11 +82,11 @@
+@@ -82,11 +68,11 @@ Running CWL workflows using Toil is easy
  
  #. To run the workflow simply enter ::
  
@@ -71,18 +67,9 @@ Description: Update docs to reflect a local install
          Hello world!
  
  To learn more about CWL, see the `CWL User Guide`_ (from where this example was
-@@ -110,7 +96,7 @@
- 
- For information on using CWL with Toil see the section :ref:`cwl`
- 
--.. _CWL User Guide: http://www.commonwl.org/v1.0/UserGuide.html
-+.. _CWL User Guide: https://www.commonwl.org/user_guide/
- 
- Running a basic WDL workflow
- ----------------------------
-@@ -118,13 +104,6 @@
+@@ -104,13 +90,6 @@ Running a basic WDL workflow
  The `Workflow Description Language`_ (WDL) is another emerging language for writing workflows that are portable across multiple workflow engines and platforms.
- Running WDL workflows using Toil is still in alpha, and currently experimental.  Toil currently supports basic workflow syntax (see :ref:`wdlSupport` for more details and examples).  Here we go over running a basic WDL helloworld workflow.
+ Running WDL workflows using Toil is still in alpha, and currently experimental.  Toil currently supports basic workflow syntax (see :ref:`wdl` for more details and examples).  Here we go over running a basic WDL helloworld workflow.
  
 -#. First ensure that Toil is installed with the
 -   ``wdl`` extra (see :ref:`extras`)::
@@ -94,7 +81,7 @@ Description: Update docs to reflect a local install
  #. Copy and paste the following code block into ``wdl-helloworld.wdl``::
  
          workflow write_simple_file {
-@@ -144,11 +123,11 @@
+@@ -130,11 +109,11 @@ Running WDL workflows using Toil is stil
  
  #. To run the workflow simply enter ::
  
@@ -108,7 +95,7 @@ Description: Update docs to reflect a local install
          Hello world!
  
  To learn more about WDL, see the main `WDL website`_ .
-@@ -175,7 +154,7 @@
+@@ -161,7 +140,7 @@ Running the example
  
  #. Run it with the default settings::
  
@@ -117,7 +104,7 @@ Description: Update docs to reflect a local install
  
     The workflow created a file called ``sortedFile.txt`` in your current directory.
     Have a look at it and notice that it contains a whole lot of sorted lines!
-@@ -192,7 +171,7 @@
+@@ -178,7 +157,7 @@ Running the example
  
  3. Run with custom options::
  
@@ -126,7 +113,7 @@ Description: Update docs to reflect a local install
  
     Here we see that we can add our own options to a Toil script. As noted above, the first two
     options, ``--numLines`` and ``--lineLength``, determine the number of lines and how many characters are in each line.
-@@ -308,7 +287,7 @@
+@@ -293,7 +272,7 @@ in addition to messages from the batch s
  with the ``--logLevel`` flag. For example, to only log ``CRITICAL`` level
  messages to the screen::
  
@@ -135,7 +122,7 @@ Description: Update docs to reflect a local install
  
  This hides most of the information we get from the Toil run. For more detail,
  we can run the pipeline with ``--logLevel=debug`` to see a comprehensive
-@@ -332,7 +311,7 @@
+@@ -317,7 +296,7 @@ example (the first line of ``down()``):
  
  When we run the pipeline, Toil will show a detailed failure log with a traceback::
  
@@ -144,7 +131,7 @@ Description: Update docs to reflect a local install
     ...
     ---TOIL WORKER OUTPUT LOG---
     ...
-@@ -354,11 +333,11 @@
+@@ -339,11 +318,11 @@ that a job store of the same name alread
  failure, the job store is preserved so that the workflow can be restarted,
  starting from the previously failed jobs. We can restart the pipeline by running ::
  
@@ -153,21 +140,44 @@ Description: Update docs to reflect a local install
  
  We can also change the number of times Toil will attempt to retry a failed job::
  
--   (venv) $ python sort.py --retryCount 2 --restart --overwriteOutput=True
-+   $ python sort.py --retryCount 2 --restart --overwriteOutput=True
+-   (venv) $ python sort.py file:jobStore --retryCount 2 --restart --overwriteOutput=True
++   $ python sort.py file:jobStore --retryCount 2 --restart --overwriteOutput=True
  
  You'll now see Toil attempt to rerun the failed job until it runs out of tries.
  ``--retryCount`` is useful for non-systemic errors, like downloading a file that
-@@ -369,7 +348,7 @@
+@@ -354,7 +333,7 @@ line 30, or remove it, and then run
  
  ::
  
--    (venv) $ python sort.py --restart --overwriteOutput=True
-+    $ python sort.py --restart --overwriteOutput=True
+-    (venv) $ python sort.py file:jobStore --restart --overwriteOutput=True
++    $ python sort.py file:jobStore --restart --overwriteOutput=True
  
  The pipeline will run successfully, and the job store will be removed on the
  pipeline's completion.
-@@ -428,7 +407,7 @@
+@@ -380,19 +359,19 @@ Also!  Remember to use the :ref:`destroy
+ 
+ #. Launch a cluster in AWS using the :ref:`launchCluster` command::
+ 
+-        (venv) $ toil launch-cluster <cluster-name> --keyPairName <AWS-key-pair-name> --leaderNodeType t2.medium --zone us-west-2a
++        $ toil launch-cluster <cluster-name> --keyPairName <AWS-key-pair-name> --leaderNodeType t2.medium --zone us-west-2a
+ 
+    The arguments ``keyPairName``, ``leaderNodeType``, and ``zone`` are required to launch a cluster.
+ 
+ #. Copy ``helloWorld.py`` to the ``/tmp`` directory on the leader node using the :ref:`rsyncCluster` command::
+ 
+-        (venv) $ toil rsync-cluster --zone us-west-2a <cluster-name> helloWorld.py :/tmp
++        $ toil rsync-cluster --zone us-west-2a <cluster-name> helloWorld.py :/tmp
+ 
+    Note that the command requires defining the file to copy as well as the target location on the cluster leader node.
+ 
+ #. Login to the cluster leader node using the :ref:`sshCluster` command::
+ 
+-        (venv) $ toil ssh-cluster --zone us-west-2a <cluster-name>
++        $ toil ssh-cluster --zone us-west-2a <cluster-name>
+ 
+    Note that this command will log you in as the ``root`` user.
+ 
+@@ -413,7 +392,7 @@ Also!  Remember to use the :ref:`destroy
  
  #. Use the :ref:`destroyCluster` command to destroy the cluster::
  
@@ -176,3 +186,105 @@ Description: Update docs to reflect a local install
  
     Note that this command will destroy the cluster leader
     node and any resources created to run the job, including the S3 bucket.
+@@ -431,17 +410,17 @@ Also!  Remember to use the :ref:`destroy
+ 
+ #. First launch a node in AWS using the :ref:`launchCluster` command::
+ 
+-      (venv) $ toil launch-cluster <cluster-name> --keyPairName <AWS-key-pair-name> --leaderNodeType t2.medium --zone us-west-2a
++      $ toil launch-cluster <cluster-name> --keyPairName <AWS-key-pair-name> --leaderNodeType t2.medium --zone us-west-2a
+ 
+ #. Copy ``example.cwl`` and ``example-job.yaml`` from the :ref:`CWL example <cwlquickstart>` to the node using
+    the :ref:`rsyncCluster` command::
+ 
+-      (venv) $ toil rsync-cluster --zone us-west-2a <cluster-name> example.cwl :/tmp
+-      (venv) $ toil rsync-cluster --zone us-west-2a <cluster-name> example-job.yaml :/tmp
++      $ toil rsync-cluster --zone us-west-2a <cluster-name> example.cwl :/tmp
++      $ toil rsync-cluster --zone us-west-2a <cluster-name> example-job.yaml :/tmp
+ 
+ #. SSH into the cluster's leader node using the :ref:`sshCluster` utility::
+ 
+-      (venv) $ toil ssh-cluster --zone us-west-2a <cluster-name>
++      $ toil ssh-cluster --zone us-west-2a <cluster-name>
+ 
+ #. Once on the leader node, it's a good idea to update and install the following::
+ 
+@@ -458,7 +437,7 @@ Also!  Remember to use the :ref:`destroy
+ 
+ #. Now run the CWL workflow::
+ 
+-      (venv) $ toil-cwl-runner --provisioner aws --jobStore aws:us-west-2a:any-name /tmp/example.cwl /tmp/example-job.yaml
++      $ toil-cwl-runner --provisioner aws --jobStore aws:us-west-2a:any-name /tmp/example.cwl /tmp/example-job.yaml
+ 
+    ..  tip::
+ 
+@@ -468,7 +447,7 @@ Also!  Remember to use the :ref:`destroy
+ 
+ #. Finally, log out of the leader node and from your local computer, destroy the cluster::
+ 
+-      (venv) $ toil destroy-cluster --zone us-west-2a <cluster-name>
++      $ toil destroy-cluster --zone us-west-2a <cluster-name>
+ 
+ 
+ .. _awscactus:
+@@ -508,7 +487,7 @@ Also!  Remember to use the :ref:`destroy
+ 
+ #. Launch a leader node using the :ref:`launchCluster` command::
+ 
+-        (venv) $ toil launch-cluster <cluster-name> --provisioner <aws, gce, azure> --keyPairName <key-pair-name> --leaderNodeType <type> --zone <zone>
++        $ toil launch-cluster <cluster-name> --provisioner <aws, gce, azure> --keyPairName <key-pair-name> --leaderNodeType <type> --zone <zone>
+ 
+ 
+    .. note::
+@@ -518,11 +497,11 @@ Also!  Remember to use the :ref:`destroy
+         When using AWS or Azure, setting the environment variable eliminates having to specify the ``--zone`` option
+         for each command. This will be supported for GCE in the future. ::
+ 
+-            (venv) $ export TOIL_AWS_ZONE=us-west-2c
++            $ export TOIL_AWS_ZONE=us-west-2c
+ 
+ #. Create appropriate directory for uploading files::
+ 
+-        (venv) $ toil ssh-cluster --provisioner <aws, gce, azure> <cluster-name>
++        $ toil ssh-cluster --provisioner <aws, gce, azure> <cluster-name>
+         $ mkdir /root/cact_ex
+         $ exit
+ 
+@@ -531,18 +510,18 @@ Also!  Remember to use the :ref:`destroy
+    `here <https://github.com/ComparativeGenomicsToolkit/cactus#seqfile-the-input-file>`__), organisms' genome sequence
+    files in FASTA format, and configuration files (e.g. blockTrim1.xml, if desired), up to the leader node::
+ 
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> pestis-short-aws-seqFile.txt :/root/cact_ex
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000169655.1_ASM16965v1_genomic.fna :/root/cact_ex
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000006645.1_ASM664v1_genomic.fna :/root/cact_ex
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000182485.1_ASM18248v1_genomic.fna :/root/cact_ex
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000013805.1_ASM1380v1_genomic.fna :/root/cact_ex
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> setup_leaderNode.sh :/root/cact_ex
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> blockTrim1.xml :/root/cact_ex
+-      (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> blockTrim3.xml :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> pestis-short-aws-seqFile.txt :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000169655.1_ASM16965v1_genomic.fna :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000006645.1_ASM664v1_genomic.fna :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000182485.1_ASM18248v1_genomic.fna :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> GCF_000013805.1_ASM1380v1_genomic.fna :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> setup_leaderNode.sh :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> blockTrim1.xml :/root/cact_ex
++      $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name> blockTrim3.xml :/root/cact_ex
+ 
+ #. Log in to the leader node::
+ 
+-        (venv) $ toil ssh-cluster --provisioner <aws, gce, azure> <cluster-name>
++        $ toil ssh-cluster --provisioner <aws, gce, azure> <cluster-name>
+ 
+ #. Set up the environment of the leader node to run Cactus::
+ 
+@@ -588,8 +567,8 @@ Also!  Remember to use the :ref:`destroy
+ 
+ #. Download the resulted output to local machine::
+ 
+-        (venv) $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name>  :/root/cact_ex/pestis_output3.hal <path-of-folder-on-local-machine>
++        $ toil rsync-cluster --provisioner <aws, gce, azure> <cluster-name>  :/root/cact_ex/pestis_output3.hal <path-of-folder-on-local-machine>
+ 
+ #. Destroy the cluster::
+ 
+-        (venv) $ toil destroy-cluster --provisioner <aws, gce, azure> <cluster-name>
++        $ toil destroy-cluster --provisioner <aws, gce, azure> <cluster-name>

debian/patches/no_galaxy_lib

 Author: Michael R. Crusoe <michael.crusoe@gmail.com>
 Description: skip galaxy-lib, not yet packaged for Debian
---- toil.orig/setup.py
-+++ toil/setup.py
-@@ -81,8 +81,7 @@
+--- a/setup.py
++++ b/setup.py
+@@ -83,8 +83,7 @@ def runSetup():
          gcs]
      cwl_reqs = [
          cwltool,

debian/patches/no_virtualenv_to_create_manpages.patch

 Author: Steffen Moeller <moeller@debian.org>
 Description: ignore requirement to use a virtualenv
-Index: toil/docs/conf.py
-===================================================================
---- toil.orig/docs/conf.py
-+++ toil/docs/conf.py
-@@ -35,8 +35,8 @@ def fetch_parent_dir(filepath, n=1):
+--- a/docs/conf.py
++++ b/docs/conf.py
+@@ -36,8 +36,8 @@ def fetch_parent_dir(filepath, n=1):
          filepath = os.path.dirname(filepath)
      return os.path.basename(filepath)
  

debian/patches/version.patch

 Author: Michael R. Crusoe <michael.crusoe@gmail.com>
 Description: allow the use of Debian packaged dependencies
---- toil.orig/setup.py
-+++ toil/setup.py
-@@ -34,8 +34,8 @@
+--- a/setup.py
++++ b/setup.py
+@@ -34,8 +34,8 @@ def runSetup():
      gcs = 'google-cloud-storage==1.6.0'
      gcs_oauth2_boto_plugin = 'gcs_oauth2_boto_plugin==1.14'
      apacheLibcloud = 'apache-libcloud==2.2.1'
--    cwltool = 'cwltool==1.0.20180820141117'
--    schemaSalad = 'schema-salad>=2.6, <3'
-+    cwltool = 'cwltool>=1.0.20180820141117'
-+    schemaSalad = 'schema-salad>=3, <3.1'
-     galaxyLib = 'galaxy-lib==17.9.3'
+-    cwltool = 'cwltool==1.0.20190228155703'
+-    schemaSalad = 'schema-salad<5,>=4.1'
++    cwltool = 'cwltool>=1.0.20181217162649'
++    schemaSalad = 'schema-salad<5,>=3.0.20181206233650'
+     galaxyLib = 'galaxy-lib==18.9.2'
      htcondor = 'htcondor>=8.6.0'
      dill = 'dill==0.2.7.1'
-@@ -150,33 +150,6 @@
+@@ -171,33 +171,6 @@ def importVersion():
      required.
      """
      import imp

docker/Dockerfile.py

@@ -21,14 +21,17 @@ sdistName = os.environ['_TOIL_SDIST_NAME']
 
 
 dependencies = ' '.join(['libffi-dev',  # For client side encryption for 'azure' extra with PyNACL
+                         'python3.6',
+                         'python3.6-dev',
                          'python-dev',  # For installing Python packages with native code
                          'python-pip',  # Bootstrap pip, but needs upgrading, see below
+                         'python3-pip',
                          'libcurl4-openssl-dev',
                          'libssl-dev',
                          'wget',
                          'curl',
                          'openssh-server',
-                         'mesos=1.0.1-2.0.93.ubuntu1404',
+                         'mesos=1.0.1-2.0.94.ubuntu1604',
                          "nodejs",  # CWL support for javascript expressions
                          'rsync',
                          'screen'])
@@ -55,24 +58,29 @@ motd = heredoc('''
 motd = ''.join(l + '\\n\\\n' for l in motd.splitlines())
 
 print(heredoc('''
-    FROM ubuntu:14.04
+    FROM ubuntu:16.04
     
-    RUN echo "deb http://repos.mesosphere.io/ubuntu/ trusty main" \
+    RUN apt-get -y update --fix-missing && apt-get -y upgrade && apt-get -y install apt-transport-https ca-certificates software-properties-common && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+    RUN echo "deb http://repos.mesosphere.io/ubuntu/ xenial main" \
         > /etc/apt/sources.list.d/mesosphere.list \
         && apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF \
-        && echo "deb http://deb.nodesource.com/node_6.x trusty main" \
+        && echo "deb http://deb.nodesource.com/node_6.x xenial main" \
         > /etc/apt/sources.list.d/nodesource.list \
-        && apt-key adv --keyserver keyserver.ubuntu.com --recv 68576280 \
-        && apt-get -y update \
-        && apt-get -y install {dependencies} \
-        && apt-get clean && rm -rf /var/lib/apt/lists/*
+        && apt-key adv --keyserver keyserver.ubuntu.com --recv 68576280
+
+    RUN add-apt-repository -y ppa:jonathonf/python-3.6
+    
+    RUN apt-get -y update --fix-missing && apt-get -y upgrade && apt-get -y install {dependencies} && apt-get clean && rm -rf /var/lib/apt/lists/*
 
     RUN mkdir /root/.ssh && \
         chmod 700 /root/.ssh
 
     ADD waitForKey.sh /usr/bin/waitForKey.sh
 
-    RUN chmod 777 /usr/bin/waitForKey.sh
+    ADD customDockerInit.sh /usr/bin/customDockerInit.sh
+
+    RUN chmod 777 /usr/bin/waitForKey.sh && chmod 777 /usr/bin/customDockerInit.sh
     
     # The stock pip is too old and can't install from sdist with extras
     RUN pip install --upgrade pip==9.0.1
@@ -89,22 +97,23 @@ print(heredoc('''
         && ln -s /home/s3am/bin/s3am /usr/local/bin/
 
     # Install statically linked version of docker client
-    RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.13.1.tgz \
+    RUN curl https://download.docker.com/linux/static/stable/x86_64/docker-18.06.1-ce.tgz \
          | tar -xvzf - --transform='s,[^/]*/,,g' -C /usr/local/bin/ \
          && chmod u+x /usr/local/bin/docker
 
     # Fix for Mesos interface dependency missing on ubuntu
     RUN pip install protobuf==3.0.0
 
-    # Move the Mesos module onto the Python path
-    RUN ln -s /usr/lib/python2.7/site-packages/mesos /usr/local/lib/python2.7/dist-packages/mesos
-
     # Fix for https://issues.apache.org/jira/browse/MESOS-3793
     ENV MESOS_LAUNCHER=posix
 
     # Fix for `screen` (https://github.com/BD2KGenomics/toil/pull/1386#issuecomment-267424561)
     ENV TERM linux
     
+    # Run bash instead of sh inside of screen
+    ENV SHELL /bin/bash
+    RUN echo "defshell -bash" > ~/.screenrc
+
     # An appliance may need to start more appliances, e.g. when the leader appliance launches the
     # worker appliance on a worker node. To support this, we embed a self-reference into the image:
     ENV TOIL_APPLIANCE_SELF {applianceSelf}

docker/customDockerInit.sh

+#!/usr/bin/env bash
+# Runs custom initialization before running the main docker service.
+# The first argument is a single string containing the custom init command.
+# The second argument is the docker service binary (e.g. mesos-slave).
+# The rest of the arguments are passed into the service.
+# All arguments are required!
+#
+# Example usage:
+# $ customDockerInit.sh 'echo "hello world"' mesos-slave --log_dir=/var/lib/mesos ...
+
+CUSTOM_INIT_COMMAND="${1}"
+SERVICE_COMMAND="${2}"
+SERVICE_COMMAND_ARGS="${@:3}"
+
+bash -c "${CUSTOM_INIT_COMMAND}"
+eval "${SERVICE_COMMAND}" "${SERVICE_COMMAND_ARGS}"

docs/appendices/architecture.rst

@@ -77,7 +77,7 @@ Job chaining
 
 The scheduling of successor jobs is partially managed by the worker, reducing the
 number of individual jobs the leader needs to process. Currently this is very
-simple: if the there is a single next successor job to run and it's resources fit within the
+simple: if the there is a single next successor job to run and its resources fit within the
 resources of the current job and closely match the resources of the current job then
 the job is run immediately on the worker without returning to the leader. Further extensions
 of this strategy are possible, but for many workflows which define a series of serial successors
@@ -143,7 +143,7 @@ Toil support for Common Workflow Language
 The CWL document and input document are loaded using the 'cwltool.load_tool'
 module.  This performs normalization and URI expansion (for example, relative
 file references are turned into absolute file URIs), validates the document
-againsted the CWL schema, initializes Python objects corresponding to major
+against the CWL schema, initializes Python objects corresponding to major
 document elements (command line tools, workflows, workflow steps), and performs
 static type checking that sources and sinks have compatible types.
 

docs/appendices/environment_vars.rst

@@ -4,7 +4,7 @@ Environment Variables
 =====================
 There are several environment variables that affect the way Toil runs.
 
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_WORKDIR                     | An absolute path to a directory where Toil will    |
 |                                  | write its temporary files. This directory must     |
 |                                  | exist on each worker node and may be set to a      |
@@ -15,7 +15,7 @@ There are several environment variables that affect the way Toil runs.
 |                                  | it defaults to ``/var/lib/mesos``. In all other    |
 |                                  | cases, the system's `standard temporary directory`_|
 |                                  | is used.                                           |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_APPLIANCE_SELF              | The fully qualified reference for the Toil         |
 |                                  | Appliance you wish to use, in the form             |
 |                                  | ``REPO/IMAGE:TAG``.                                |
@@ -24,24 +24,24 @@ There are several environment variables that affect the way Toil runs.
 |                                  | options. Note that since Docker defaults to        |
 |                                  | Dockerhub repos, only quay.io repos need to        |
 |                                  | specify their registry.                            |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_DOCKER_REGISTRY             | The URL of the registry of the Toil Appliance      |
 |                                  | image you wish to use. Docker will use Dockerhub   |
 |                                  | by default, but the quay.io registry is also       |
 |                                  | very popular and easily specifiable by settting    |
 |                                  | this option to ``quay.io``.                        |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_DOCKER_NAME                 | The name of the Toil Appliance image you           |
 |                                  | wish to use. Generally this is simply ``toil`` but |
 |                                  | this option is provided to override this,          |
 |                                  | since the image can be built with arbitrary names. |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_AWS_ZONE                    | The EC2 zone to provision nodes in if using        |
 |                                  | Toil's provisioner.                                |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_AWS_AMI                     | ID of the AMI to use in node provisioning. If in   |
 |                                  | doubt, don't set this variable.                    |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_AWS_NODE_DEBUG              | Determines whether to preserve nodes that have     |
 |                                  | failed health checks. If set to ``True``, nodes    |
 |                                  | that fail EC2 health checks won't immediately be   |
@@ -51,24 +51,24 @@ There are several environment variables that affect the way Toil runs.
 |                                  | also be left behind by necessity as it cannot be   |
 |                                  | deleted until all associated nodes have been       |
 |                                  | terminated.                                        |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_AZURE_ZONE                  | A specified region for provisioning instances.     |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_SLURM_ARGS                  | Arguments for sbatch for the slurm batch system.   |
 |                                  | Do not pass CPU or memory specifications here.     |
 |                                  | Instead, define resource requirements for the job. |
 |                                  | There is no default value for this variable.       |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_GRIDENGINE_ARGS             | Arguments for qsub for the gridengine batch        |
 |                                  | system. Do not pass CPU or memory specifications   |
 |                                  | here. Instead, define resource requirements for    |
 |                                  | the job. There is no default value for this        |
 |                                  | variable.                                          |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_GRIDENGINE_PE               | Parallel environment arguments for qsub and for    |
 |                                  | the gridengine batch system. There is no default   |
 |                                  | value for this variable.                           |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_TORQUE_ARGS                 | Arguments for qsub for the Torque batch system.    |
 |                                  | Do not pass CPU or memory specifications here.     |
 |                                  | Instead, define extra parameters for the job such  |
@@ -76,7 +76,7 @@ There are several environment variables that affect the way Toil runs.
 |                                  | Use TOIL_TORQUE_REQS to pass extra values for the  |
 |                                  | -l resource requirements parameter.                |
 |                                  | There is no default value for this variable.       |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_TORQUE_REQS                 | Arguments for the resource requirements for Torque |
 |                                  | batch system. Do not pass CPU or memory            |
 |                                  | specifications here. Instead, define extra resource|
@@ -84,11 +84,29 @@ There are several environment variables that affect the way Toil runs.
 |                                  | argument to qsub. Example:                         |
 |                                  | walltime=2:00:00,file=50gb                         |
 |                                  | There is no default value for this variable.       |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
 | TOIL_LSF_ARGS                    | Additional arguments for the LSF's bsub command.   |
 |                                  | Instead, define extra parameters for the job such  |
-|                        | as queue. Example: -q medium                       |
+|                                  | as queue. Example: -q medium.                      |
+|                                  | There is no default value for this variable.       |
++----------------------------------+----------------------------------------------------+
+| TOIL_HTCONDOR_PARAMS             | Additional parameters to include in the HTCondor   |
+|                                  | submit file passed to condor_submit. Do not pass   |
+|                                  | CPU or memory specifications here. Instead define  |
+|                                  | extra parameters which may be required by HTCondor.|
+|                                  | This variable is parsed as a semicolon-separated   |
+|                                  | string of ``parameter = value`` pairs. Example:    |
+|                                  | ``requirements = TARGET.has_sse4_2 == true;        |
+|                                  | accounting_group = test``.                         |
 |                                  | There is no default value for this variable.       |
-+------------------------+----------------------------------------------------+
++----------------------------------+----------------------------------------------------+
+| TOIL_CUSTOM_DOCKER_INIT_COMMAND  | Any custom bash command to run in the Toil docker  |
+|                                  | container prior to running the Toil services.      |
+|                                  | Can be used for any custom initialization in the   |
+|                                  | worker and/or primary nodes such as private docker |
+|                                  | docker authentication. Example for AWS ECR:        |
+|                                  | ``pip install awscli && eval $(aws ecr get-login   |
+|                                  | --no-include-email --region us-east-1)``.          |
++----------------------------------+----------------------------------------------------+
 
 .. _standard temporary directory: https://docs.python.org/2/library/tempfile.html#tempfile.gettempdir