Commit 05e4af94 authored by Michael R. Crusoe's avatar Michael R. Crusoe 🏳️‍🌈
Browse files

New upstream version 3.21.0

parent 0e389f5c

.gitlab-ci.yml

+138 −4
Original line number Diff line number Diff line 
before_script:

  - docker info

  - cat /etc/hosts

  - export PYTHONIOENCODING=utf-8

after_script:

  # We need to clean up any files that Toil may have made via Docker that

  # aren't deletable by the Gitlab user. If we don't do this, Gitlab will try
@@ -11,17 +12,150 @@ after_script: 




stages:

  - main_tests

  - test

  - integration





py2-job:

# Python2.7

py2_batch_systems:

  stage: test

  script:

    - pwd

    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor && PYTHONIOENCODING=utf-8 make test

    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/batchSystems/batchSystemTest.py

    - python -m pytest src/toil/test/mesos/MesosDataStructuresTest.py



py3-job:

py2_cwl:

  stage: test

  script:

    - pwd

    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor && PYTHONIOENCODING=utf-8 make test
 
    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/cwl/cwlTest.py



py2_wdl:

  stage: test

  script:

    - pwd

    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/wdl/toilwdlTest.py



py2_jobstore_and_provisioning:

  stage: test

  script:

    - pwd

    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/sort/sortTest.py

    - python -m pytest src/toil/test/provisioners/aws/awsProvisionerTest.py

    # - python -m pytest src/toil/test/provisioners/azureProvisionerTest.py  # disabled and no longer maintained

    - python -m pytest src/toil/test/provisioners/clusterScalerTest.py

    - python -m pytest src/toil/test/provisioners/gceProvisionerTest.py



py2_main:

  stage: main_tests

  script:

    - pwd

    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/src

    - python -m pytest src/toil/test/utils



py2_integration_jobstore:

  stage: integration

  script:

    - pwd

    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata && apt install -y awscli jq

    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli

    - export TOIL_TEST_INTEGRATIVE=True

    - export TOIL_AWS_KEYNAME=id_rsa

    - export TOIL_AWS_ZONE=us-west-2a

    - python setup_gitlab_ssh.py

    - mkdir ~/.aws

    - echo -e $(aws secretsmanager get-secret-value --secret-id allspark/runner/credentials --region us-west-2 | jq -r .SecretString) > ~/.aws/credentials

    - python -m pytest src/toil/test/jobStores/jobStoreTest.py



py2_integration_sort:

  stage: integration

  script:

    - pwd

    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata && apt install -y awscli jq

    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli

    - export TOIL_TEST_INTEGRATIVE=True

    - export TOIL_AWS_KEYNAME=id_rsa

    - export TOIL_AWS_ZONE=us-west-2a

    - python setup_gitlab_ssh.py

    - mkdir ~/.aws

    - echo -e $(aws secretsmanager get-secret-value --secret-id allspark/runner/credentials --region us-west-2 | jq -r .SecretString) > ~/.aws/credentials

    - python -m pytest src/toil/test/sort/sortTest.py

    - python -m pytest src/toil/test/provisioners/clusterScalerTest.py



#py2_integration_provisioner:

#  stage: integration

#  script:

#    - pwd

#    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata && apt install -y awscli jq

#    - virtualenv -p python2.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli

#    - export TOIL_TEST_INTEGRATIVE=True

#    - export TOIL_AWS_KEYNAME=id_rsa

#    - export TOIL_AWS_ZONE=us-west-2a

#    - python setup_gitlab_ssh.py

#    - mkdir ~/.aws

#    - echo -e $(aws secretsmanager get-secret-value --secret-id allspark/runner/credentials --region us-west-2 | jq -r .SecretString) > ~/.aws/credentials

#    - python -m pytest src/toil/test/provisioners/aws/awsProvisionerTest.py





# Python3.6

py3_batch_systems:

  stage: test

  script:

    - pwd

    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/batchSystems/batchSystemTest.py

    - python -m pytest src/toil/test/mesos/MesosDataStructuresTest.py



py3_cwl:

  stage: test

  script:

    - pwd

    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/cwl/cwlTest.py



py3_wdl:

  stage: test

  script:

    - pwd

    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/wdl/toilwdlTest.py



py3_jobstore_and_provisioning:

  stage: test

  script:

    - pwd

    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/jobStores/jobStoreTest.py

    - python -m pytest src/toil/test/sort/sortTest.py

    - python -m pytest src/toil/test/provisioners/aws/awsProvisionerTest.py

    # - python -m pytest src/toil/test/provisioners/azureProvisionerTest.py  # disabled and no longer maintained

    - python -m pytest src/toil/test/provisioners/clusterScalerTest.py

    - python -m pytest src/toil/test/provisioners/gceProvisionerTest.py



py3_main:

  stage: main_tests

  script:

    - pwd

    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor

    - python -m pytest src/toil/test/src

    - python -m pytest src/toil/test/utils



#py3_integration:

#  stage: integration

#  script:

#    - pwd

#    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata && apt install -y awscli jq

#    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli

#    - export TOIL_TEST_INTEGRATIVE=True

#    - export TOIL_AWS_KEYNAME=id_rsa

#    - export TOIL_AWS_ZONE=us-west-2a

#    - python setup_gitlab_ssh.py

#    - mkdir ~/.aws

#    - echo -e $(aws secretsmanager get-secret-value --secret-id allspark/runner/credentials --region us-west-2 | jq -r .SecretString) > ~/.aws/credentials

#    - python -m pytest src/toil/test/jobStores/jobStoreTest.py

Makefile

+21 −57
Original line number Diff line number Diff line
@@ -59,9 +59,6 @@ test. This test does not capture terminal output. 


The 'integration_test' target is the same as the previous except that it does capture output.



The 'test_parallel' target runs Toil's unit tests in parallel and generates an XML test report

from the results. It is designed to be used only in Jenkins.



The 'pypi' target publishes the current commit of Toil to PyPI after enforcing that the working

copy and the index are clean.
@@ -88,6 +85,10 @@ export help 
help:

	@printf "$$help"











# This Makefile uses bash features like printf and <()

SHELL=bash

python=python
@@ -102,14 +103,10 @@ dist_version:=$(shell $(python) version_template.py distVersion) 
sdist_name:=toil-$(dist_version).tar.gz



docker_tag:=$(shell $(python) version_template.py dockerTag)

default_docker_registry:=$(shell $(python) version_template.py dockerRegistry)

default_docker_registry:=quay.io/ucsc_cgl

docker_path:=$(strip $(shell which docker))



ifdef docker_registry

    export TOIL_DOCKER_REGISTRY?=$(docker_registry)

else

    export TOIL_DOCKER_REGISTRY?=$(default_docker_registry)

endif

export TOIL_DOCKER_REGISTRY?=$(shell $(python) version_template.py dockerRegistry)

export TOIL_DOCKER_NAME?=$(shell $(python) version_template.py dockerName)

export TOIL_APPLIANCE_SELF:=$(TOIL_DOCKER_REGISTRY)/$(TOIL_DOCKER_NAME):$(docker_tag)
@@ -122,12 +119,14 @@ endif 


develop: check_venv

	$(pip) install -e .$(extras)



clean_develop: check_venv

	- $(pip) uninstall -y toil

	- rm -rf src/*.egg-info

	- rm src/toil/version.py



sdist: dist/$(sdist_name)



dist/$(sdist_name): check_venv

	@test -f dist/$(sdist_name) && mv dist/$(sdist_name) dist/$(sdist_name).old || true

	$(python) setup.py sdist
@@ -137,48 +136,35 @@ dist/$(sdist_name): check_venv 
	         && printf "$(cyan)No significant changes to sdist, reinstating backup.$(normal)\n" \

	         || rm dist/$(sdist_name).old ) \

	    || true



clean_sdist:

	- rm -rf dist

	- rm src/toil/version.py





# We always claim to be Travis, so that local test runs will not skip Travis tests.

# Gitlab doesn't run tests via the Makefile.



# This target will skip building docker and all docker based tests

test_offline: check_venv check_build_reqs

	@printf "$(cyan)All docker related tests will be skipped.$(normal)\n"

	TOIL_SKIP_DOCKER=True \

	TRAVIS=true \

	    $(python) -m pytest $(pytest_args_local) $(tests_local)



# The auto-deployment test needs the docker appliance

test: check_venv check_build_reqs docker

	TOIL_APPLIANCE_SELF=$(docker_registry)/$(docker_base_name):$(docker_tag) \

	TRAVIS=true \

	    $(python) -m pytest --cov=toil $(pytest_args_local) $(tests)



# For running integration tests locally in series (uses the -s argument for pyTest)

integration_test_local: check_venv check_build_reqs sdist push_docker

	TOIL_TEST_INTEGRATIVE=True \

	TRAVIS=true \

	    $(python) run_tests.py --local integration-test $(tests)



# These two targets are for backwards compatibility but will be removed shortly

# FIXME when they are removed add check_running_on_jenkins to the jenkins targets

test_parallel: jenkins_test_parallel



integration_test: jenkins_test_integration



# This target is designed only for use on Jenkins

jenkins_test_parallel: check_venv check_build_reqs docker

	$(python) run_tests.py test $(tests)



# This target is designed only for use on Jenkins

jenkins_test_integration: check_venv check_build_reqs sdist push_docker

	TOIL_TEST_INTEGRATIVE=True $(python) run_tests.py integration-test $(tests)





pypi: check_venv check_clean_working_copy check_running_on_jenkins

	$(pip) install setuptools --upgrade

	$(python) setup.py egg_info sdist bdist_egg upload

clean_pypi:

	- rm -rf build/



test_integration: check_venv check_build_reqs docker

	TRAVIS=true \

	    $(python) run_tests.py integration-test $(tests)



ifdef TOIL_DOCKER_REGISTRY
@@ -197,7 +183,6 @@ define tag_docker 
	@printf "$(green)Tagged appliance image $1 as $2.$(normal)\n"

endef





docker: docker/Dockerfile

	@set -ex \

	; cd docker \
@@ -230,13 +215,6 @@ clean_docker: 
	-rm docker/Dockerfile docker/$(sdist_name)

	-docker rmi $(docker_image):$(docker_tag)



obliterate_docker: clean_docker

	-@set -x \

	; docker images $(docker_image) \

	    | tail -n +2 | awk '{print $$1 ":" $$2}' | uniq \

	    | xargs docker rmi

	-docker images -qf dangling=true | xargs docker rmi



push_docker: docker check_docker_registry

	for i in $$(seq 1 5); do docker push $(docker_image):$(docker_tag) && break || sleep 60; done

	for i in $$(seq 1 5); do docker push $(grafana_image):$(docker_tag) && break || sleep 60; done
@@ -254,27 +232,23 @@ endif 
docs: check_venv check_build_reqs

	# Strange, but seemingly benign Sphinx warning floods stderr if not filtered:

	cd docs && make html



clean_docs: check_venv

	- cd docs && make clean





clean: clean_develop clean_sdist clean_pypi clean_docs





check_build_reqs:

	@$(python) -c 'import mock; import pytest' \

		|| ( printf "$(red)Build requirements are missing. Run 'make prepare' to install them.$(normal)\n" ; false )





prepare: check_venv

	$(pip) install mock==1.0.1 pytest==4.3.1 pytest-cov==2.6.1 stubserver==1.0.1 pytest-timeout==1.3.3 cwltest





check_venv:

	@$(python) -c 'import sys, os; sys.exit( int( 0 if "VIRTUAL_ENV" in os.environ else 1 ) )' \

		|| ( printf "$(red)A virtualenv must be active.$(normal)\n" ; false )





check_clean_working_copy:

	@printf "$(green)Checking if your working copy is clean ...$(normal)\n"

	@git diff --exit-code > /dev/null \
@@ -286,33 +260,23 @@ check_clean_working_copy: 
			; git ls-files --other --exclude-standard --directory \

			; false )





check_running_on_jenkins:

	@printf "$(green)Checking if running on Jenkins ...$(normal)\n"

	@test -n "$$BUILD_NUMBER" \

		|| ( printf "$(red)This target should only be invoked on Jenkins.$(normal)\n" ; false )





check_docker_registry:

	@test "$(default_docker_registry)" != "$(TOIL_DOCKER_REGISTRY)" || test -n "$$BUILD_NUMBER" \

		|| ( printf '$(red)Please set TOIL_DOCKER_REGISTRY to a value other than \

	$(default_docker_registry) and ensure that you have permissions to push \

	to that registry. Only CI builds should push to $(default_docker_registry).$(normal)\n' ; false )





check_cpickle:

	# fail if cPickle.dump(s) called without HIGHEST_PROTOCOL

	# https://github.com/BD2KGenomics/toil/issues/1503

	! find src -iname '*.py' | xargs grep 'cPickle.dump' | grep --invert-match HIGHEST_PROTOCOL





.PHONY: help \

		prepare \

		check_cpickle \

		develop clean_develop \

		sdist clean_sdist \

		test test_offline test_parallel integration_test \

		jenkins_test_parallel jenkins_test_integration \

		pypi clean_pypi \

		docs clean_docs \

		clean \

README.rst

+3 −3
Original line number Diff line number Diff line 
ATTENTION: Toil has moved from https://github.com/BD2KGenomics/toil to https://github.com/DataBiosphere/toil as of July 5th, 2018.

ATTENTION: Toil will be dropping python 2.7 support on January 1, 2020 when python 2.7 itself is scheduled to die.  This is when the last release of python 2.7 compatible toil will be.



Toil is a scalable, efficient, cross-platform (Linux & macOS) pipeline management system,

written entirely in Python, and designed around the principles of functional

programming.



Our next scheduled release is May 24, 2019.



* Check the `website`_ for a description of Toil and its features.

* Full documentation for the latest stable release can be found at

  `Read the Docs`_.
@@ -21,3 +19,5 @@ Our next scheduled release is May 24, 2019. 
.. image:: https://badges.gitter.im/bd2k-genomics-toil/Lobby.svg

   :alt: Join the chat at https://gitter.im/bd2k-genomics-toil/Lobby

   :target: https://gitter.im/bd2k-genomics-toil/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge



Note: Toil moved from https://github.com/BD2KGenomics/toil to https://github.com/DataBiosphere/toil on July 5th, 2018.

docker/Dockerfile.py

+31 −4
Original line number Diff line number Diff line
@@ -34,7 +34,15 @@ dependencies = ' '.join(['libffi-dev', # For client side encryption for 'azure' 
                         'mesos=1.0.1-2.0.94.ubuntu1604',

                         "nodejs",  # CWL support for javascript expressions

                         'rsync',

                         'screen'])

                         'screen',

                         'build-essential', # We need a build environment to build Singularity 3.

                         'uuid-dev',

                         'libgpgme11-dev',

                         'libseccomp-dev',

                         'pkg-config',

                         'squashfs-tools',

                         'cryptsetup',

                         'git'])





def heredoc(s):
@@ -71,7 +79,26 @@ print(heredoc(''' 


    RUN add-apt-repository -y ppa:jonathonf/python-3.6

    

    RUN apt-get -y update --fix-missing && apt-get -y upgrade && apt-get -y install {dependencies} && apt-get clean && rm -rf /var/lib/apt/lists/*

    RUN apt-get -y update --fix-missing && \

        DEBIAN_FRONTEND=noninteractive apt-get -y upgrade && \

        DEBIAN_FRONTEND=noninteractive apt-get -y install {dependencies} && \

        apt-get clean && \

        rm -rf /var/lib/apt/lists/*



    RUN wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz && \

        tar xvf go1.13.3.linux-amd64.tar.gz && \

        mv go/bin/* /usr/bin/ && \

        mv go /usr/local/

        

    RUN mkdir -p $(go env GOPATH)/src/github.com/sylabs && \

        cd $(go env GOPATH)/src/github.com/sylabs && \

        git clone https://github.com/sylabs/singularity.git && \

        cd singularity && \

        git checkout v3.4.2 && \

        ./mconfig && \

        cd ./builddir && \

        make -j4 && \

        make install

    

    RUN mkdir /root/.ssh && \

        chmod 700 /root/.ssh

docs/appendices/architecture.rst

+2 −2
Original line number Diff line number Diff line
@@ -186,8 +186,8 @@ the single_job_executor from CWLTool, providing a Toil-specific constructor for 
filesystem access, and overriding the default PathMapper to use ToilPathMapper.



The ToilPathMapper keeps track of a file's symbolic identifier (the Toil

FileStore token), its local path on the host (the value returned by

readGlobalFile) and the the location of the file inside the Docker container.

FileID), its local path on the host (the value returned by readGlobalFile) and

the the location of the file inside the Docker container.



After executing single_job_executor from CWLTool, it gets back the output

object and status.  If the underlying job failed, raise an exception.  Files