Skip to content

Commits on Source 3

Show whitespace changes
Inline Side-by-side
Makefile
View file @ b5a53e1c
......@@ -34,12 +34,6 @@ VERSION=1.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
--format=format:%cd --date=format-local:%Y%m%d%H%M%S)
mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
UNAME_S=$(shell uname -s)
ifeq ($(UNAME_S),Linux)
nproc=$(shell nproc)
endif
ifeq ($(UNAME_S),Darwin)
nproc=$(shell sysctl -n hw.physicalcpu)
endif
## all : default task
all:
......@@ -120,11 +114,11 @@ format: autopep8
## pylint : run static code analysis on Python code
pylint: $(PYSOURCES)
pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
$^ -j$(nproc)|| true
$^ -j0|| true
pylint_report.txt: ${PYSOURCES}
pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
$^ -j$(nproc)> $@ || true
$^ -j0> $@ || true
diff_pylint_report: pylint_report.txt
diff-quality --violations=pylint pylint_report.txt
......@@ -154,11 +148,11 @@ diff-cover.html: coverage.xml
## test : run the ${MODULE} test suite
test: $(pysources)
python setup.py test --addopts "-n$(nproc) --dist=loadfile"
python setup.py test --addopts "-n auto --dist=loadfile"
## testcov : run the ${MODULE} test suite and collect coverage
testcov: $(pysources)
python setup.py test --addopts "--cov cwltool -n$(nproc) --dist=loadfile"
python setup.py test --addopts "--cov cwltool -n auto --dist=loadfile"
sloccount.sc: ${PYSOURCES} Makefile
sloccount --duplicates --wide --details $^ > $@
......
PKG-INFO
View file @ b5a53e1c
Metadata-Version: 2.1
Name: cwltool
Version: 1.0.20190815141648
Version: 1.0.20190915164430
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Author: Common workflow language working group
......@@ -13,6 +13,10 @@ Description: ==================================================================
CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Coverage Status| |Downloads|
|CommandLineTool Support| |DockerRequirement Support| |EnvVarRequirement Support| |ExpressionTool Support|
|InitialWorkDirRequirement Support| |InlineJavascriptRequirement Support| |MultipleInputRequirement Support| |Core Support|
|ResourceRequirement Support| |ScatterRequirement Support| |SchemaDefRequirement Support| |ShellCommandequirement Support|
|StepInputRequirement Support| |SubWorkflowRequirement Support| |Workflow Support|
.. |Conformance Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
:target: https://ci.commonwl.org/job/cwltool-conformance/
......@@ -29,6 +33,52 @@ Description: ==================================================================
.. |Downloads| image:: https://pepy.tech/badge/cwltool/month
:target: https://pepy.tech/project/cwltool
.. |CommandLineTool Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/command_line_tool.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |DockerRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/docker.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |EnvVarRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/env_var.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ExpressionTool Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/expression_tool.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |InitialWorkDirRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/initial_work_dir.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |InlineJavascriptRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/inline_javascript.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |MultipleInputRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/multiple_input.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |Core Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/required.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ResourceRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/resource.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ScatterRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/scatter.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |SchemaDefRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/schema_def.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ShellCommandequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/shell_command.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |StepInputRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/step_input.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |SubWorkflowRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/subworkflow.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |Workflow Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/workflow.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
This is the reference implementation of the Common Workflow Language. It is
intended to be feature complete and provide comprehensive validation of CWL
files as well as provide other tools related to working with CWL.
......@@ -53,15 +103,9 @@ Description: ==================================================================
apt-get install cwltool
For MacOS X, other UNIXes or Windows packages prepared by the Bioconda project. Please follow instructions of Bioconda (https://bioconda.github.io/) for its installation, then perform:
.. code:: bash
conda install -c bioconda cwltool
Under the hood, conda setups virtual environments before installing `cwltool` to
avoid conflicting versions of the same library. When installing cwltool directly,
it is recommended to do the same manually:
Otherwise, to
avoid conflicting versions of the same library,
it is recommended to do the following:
.. code:: bash
......
README.rst
View file @ b5a53e1c
......@@ -4,6 +4,10 @@ Common Workflow Language tool description reference implementation
CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Coverage Status| |Downloads|
|CommandLineTool Support| |DockerRequirement Support| |EnvVarRequirement Support| |ExpressionTool Support|
|InitialWorkDirRequirement Support| |InlineJavascriptRequirement Support| |MultipleInputRequirement Support| |Core Support|
|ResourceRequirement Support| |ScatterRequirement Support| |SchemaDefRequirement Support| |ShellCommandequirement Support|
|StepInputRequirement Support| |SubWorkflowRequirement Support| |Workflow Support|
.. |Conformance Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
:target: https://ci.commonwl.org/job/cwltool-conformance/
......@@ -20,6 +24,52 @@ CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Cov
.. |Downloads| image:: https://pepy.tech/badge/cwltool/month
:target: https://pepy.tech/project/cwltool
.. |CommandLineTool Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/command_line_tool.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |DockerRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/docker.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |EnvVarRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/env_var.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ExpressionTool Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/expression_tool.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |InitialWorkDirRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/initial_work_dir.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |InlineJavascriptRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/inline_javascript.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |MultipleInputRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/multiple_input.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |Core Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/required.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ResourceRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/resource.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ScatterRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/scatter.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |SchemaDefRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/schema_def.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ShellCommandequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/shell_command.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |StepInputRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/step_input.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |SubWorkflowRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/subworkflow.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |Workflow Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/workflow.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
This is the reference implementation of the Common Workflow Language. It is
intended to be feature complete and provide comprehensive validation of CWL
files as well as provide other tools related to working with CWL.
......@@ -44,15 +94,9 @@ Your operating system may offer cwltool directly. For [Debian](https://tracker.d
apt-get install cwltool
For MacOS X, other UNIXes or Windows packages prepared by the Bioconda project. Please follow instructions of Bioconda (https://bioconda.github.io/) for its installation, then perform:
.. code:: bash
conda install -c bioconda cwltool
Under the hood, conda setups virtual environments before installing `cwltool` to
avoid conflicting versions of the same library. When installing cwltool directly,
it is recommended to do the same manually:
Otherwise, to
avoid conflicting versions of the same library,
it is recommended to do the following:
.. code:: bash
......
cwltool.egg-info/PKG-INFO
View file @ b5a53e1c
Metadata-Version: 2.1
Name: cwltool
Version: 1.0.20190815141648
Version: 1.0.20190915164430
Summary: Common workflow language reference implementation
Home-page: https://github.com/common-workflow-language/cwltool
Author: Common workflow language working group
......@@ -13,6 +13,10 @@ Description: ==================================================================
CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Coverage Status| |Downloads|
|CommandLineTool Support| |DockerRequirement Support| |EnvVarRequirement Support| |ExpressionTool Support|
|InitialWorkDirRequirement Support| |InlineJavascriptRequirement Support| |MultipleInputRequirement Support| |Core Support|
|ResourceRequirement Support| |ScatterRequirement Support| |SchemaDefRequirement Support| |ShellCommandequirement Support|
|StepInputRequirement Support| |SubWorkflowRequirement Support| |Workflow Support|
.. |Conformance Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
:target: https://ci.commonwl.org/job/cwltool-conformance/
......@@ -29,6 +33,52 @@ Description: ==================================================================
.. |Downloads| image:: https://pepy.tech/badge/cwltool/month
:target: https://pepy.tech/project/cwltool
.. |CommandLineTool Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/command_line_tool.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |DockerRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/docker.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |EnvVarRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/env_var.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ExpressionTool Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/expression_tool.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |InitialWorkDirRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/initial_work_dir.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |InlineJavascriptRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/inline_javascript.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |MultipleInputRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/multiple_input.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |Core Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/required.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ResourceRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/resource.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ScatterRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/scatter.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |SchemaDefRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/schema_def.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |ShellCommandequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/shell_command.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |StepInputRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/step_input.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |SubWorkflowRequirement Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/subworkflow.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
.. |Workflow Support| image:: https://flat.badgen.net/https/raw.githubusercontent.com/common-workflow-language/conformance/master/cwltool/cwl_v1.0/cwltool_latest/workflow.json
:target: https://ci.commonwl.org/job/cwltool-conformance/
This is the reference implementation of the Common Workflow Language. It is
intended to be feature complete and provide comprehensive validation of CWL
files as well as provide other tools related to working with CWL.
......@@ -53,15 +103,9 @@ Description: ==================================================================
apt-get install cwltool
For MacOS X, other UNIXes or Windows packages prepared by the Bioconda project. Please follow instructions of Bioconda (https://bioconda.github.io/) for its installation, then perform:
.. code:: bash
conda install -c bioconda cwltool
Under the hood, conda setups virtual environments before installing `cwltool` to
avoid conflicting versions of the same library. When installing cwltool directly,
it is recommended to do the same manually:
Otherwise, to
avoid conflicting versions of the same library,
it is recommended to do the following:
.. code:: bash
......
cwltool.egg-info/SOURCES.txt
View file @ b5a53e1c
......@@ -178,6 +178,7 @@ cwltool/schemas/v1.1/salad/schema_salad/metaschema/vocab_res_src.yml
tests/2.fasta
tests/2.fastq
tests/__init__.py
tests/arg-empty-prefix-separate-false.cwl
tests/bundle-context.jsonld
tests/debian_image_id.cwl
tests/echo-job.yaml
......@@ -186,6 +187,7 @@ tests/echo-position-expr.cwl
tests/echo.cwl
tests/echo_broken_outputs.cwl
tests/env.cwl
tests/env2.cwl
tests/listing-job.yml
tests/listing2-job.yml
tests/non_portable.cwl
......
cwltool/argparser.py
View file @ b5a53e1c
......@@ -394,14 +394,16 @@ class DirectoryAppendAction(FSAppendAction):
def add_argument(toolparser, name, inptype, records, description="",
default=None):
# type: (argparse.ArgumentParser, Text, Any, List[Text], Text, Any) -> None
default=None, input_required=True):
# type: (argparse.ArgumentParser, Text, Any, List[Text], Text, Any, bool) -> None
if len(name) == 1:
flag = "-"
else:
flag = "--"
required = default is None
# if input_required is false, don't make the command line
# parameter required.
required = default is None and input_required
if isinstance(inptype, MutableSequence):
if inptype[0] == "null":
required = False
......@@ -462,8 +464,8 @@ def add_argument(toolparser, name, inptype, records, description="",
default=default, **typekw)
def generate_parser(toolparser, tool, namemap, records):
# type: (argparse.ArgumentParser, Process, Dict[Text, Text], List[Text]) -> argparse.ArgumentParser
def generate_parser(toolparser, tool, namemap, records, input_required=True):
# type: (argparse.ArgumentParser, Process, Dict[Text, Text], List[Text], bool) -> argparse.ArgumentParser
toolparser.add_argument("job_order", nargs="?", help="Job input json file")
namemap["job_order"] = "job_order"
......@@ -473,6 +475,6 @@ def generate_parser(toolparser, tool, namemap, records):
inptype = inp["type"]
description = inp.get("doc", "")
default = inp.get("default", None)
add_argument(toolparser, name, inptype, records, description, default)
add_argument(toolparser, name, inptype, records, description, default, input_required)
return toolparser
cwltool/builder.py
View file @ b5a53e1c
......@@ -426,7 +426,7 @@ class Builder(HasReqsHints):
if sep:
args.extend([prefix, self.tostr(j)])
else:
args.append("" if not prefix else prefix + self.tostr(j))
args.append(self.tostr(j) if prefix is None else prefix + self.tostr(j))
return [a for a in args if a is not None]
......
cwltool/executors.py
View file @ b5a53e1c
......@@ -159,7 +159,8 @@ class SingleJobExecutor(JobExecutor):
user_provenance=False,
orcid=runtime_context.orcid,
# single tool execution, so RO UUID = wf UUID = tool UUID
run_uuid=runtime_context.research_obj.ro_uuid)
run_uuid=runtime_context.research_obj.ro_uuid,
fsaccess=runtime_context.make_fs_access(''))
process.parent_wf = process.provenance_object
jobiter = process.job(job_order_object, self.output_callback,
runtime_context)
......@@ -178,6 +179,7 @@ class SingleJobExecutor(JobExecutor):
prov_obj = job.prov_obj
if prov_obj:
runtime_context.prov_obj = prov_obj
prov_obj.fsaccess = runtime_context.make_fs_access('')
prov_obj.evaluate(
process, job, job_order_object,
runtime_context.research_obj)
......
cwltool/job.py
View file @ b5a53e1c
......@@ -621,6 +621,16 @@ class ContainerCommandLineJob(with_metaclass(ABCMeta, JobBase)):
img_id = str(docker_req["dockerImageId"])
elif 'dockerPull' in docker_req:
img_id = str(docker_req["dockerPull"])
cmd = [user_space_docker_cmd, "pull", img_id]
_logger.info(Text(cmd))
try:
subprocess.check_call(cmd, stdout=sys.stderr)
except OSError:
raise WorkflowException(SourceLine(docker_req).makeError(
"Either Docker container {} is not available with "
"user space docker implementation {} or {} is missing "
"or broken.".format(img_id, user_space_docker_cmd,
user_space_docker_cmd)))
else:
raise WorkflowException(SourceLine(docker_req).makeError(
"Docker image must be specified as 'dockerImageId' or "
......@@ -695,7 +705,10 @@ class ContainerCommandLineJob(with_metaclass(ABCMeta, JobBase)):
time.sleep(1)
if process.returncode is not None:
if cleanup_cidfile:
try:
os.remove(cidfile)
except OSError as exc:
_logger.warn("Ignored error cleaning up Docker cidfile: %s", exc)
return
try:
with open(cidfile) as cidhandle:
......@@ -705,12 +718,16 @@ class ContainerCommandLineJob(with_metaclass(ABCMeta, JobBase)):
max_mem = psutil.virtual_memory().total
tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)
stats_file = tempfile.NamedTemporaryFile(prefix=tmp_prefix, dir=tmp_dir)
try:
with open(stats_file.name, mode="w") as stats_file_handle:
stats_proc = subprocess.Popen(
['docker', 'stats', '--no-trunc', '--format', '{{.MemPerc}}',
cid], stdout=stats_file_handle, stderr=subprocess.DEVNULL)
process.wait()
stats_proc.kill()
except OSError as exc:
_logger.warn("Ignored error with docker stats: %s", exc)
return
max_mem_percent = 0
with open(stats_file.name, mode="r") as stats:
for line in stats:
......
cwltool/main.py
View file @ b5a53e1c
......@@ -285,14 +285,15 @@ def init_job_order(job_order_object, # type: Optional[MutableMapping[Text
relative_deps=False, # type: bool
make_fs_access=StdFsAccess, # type: Callable[[Text], StdFsAccess]
input_basedir="", # type: Text
secret_store=None # type: Optional[SecretStore]
secret_store=None, # type: Optional[SecretStore]
input_required=True # type: bool
): # type: (...) -> MutableMapping[Text, Any]
secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets")
if job_order_object is None:
namemap = {} # type: Dict[Text, Text]
records = [] # type: List[Text]
toolparser = generate_parser(
argparse.ArgumentParser(prog=args.workflow), process, namemap, records)
argparse.ArgumentParser(prog=args.workflow), process, namemap, records, input_required)
if args.tool_help:
toolparser.print_help()
exit(0)
......@@ -490,7 +491,8 @@ def main(argsl=None, # type: Optional[List[str]]
custom_schema_callback=None, # type: Optional[Callable[[], None]]
executor=None, # type: Optional[JobExecutor]
loadingContext=None, # type: Optional[LoadingContext]
runtimeContext=None # type: Optional[RuntimeContext]
runtimeContext=None, # type: Optional[RuntimeContext]
input_required=True # type: bool
): # type: (...) -> int
if not stdout: # force UTF-8 even if the console is configured differently
if (hasattr(sys.stdout, "encoding")
......@@ -598,6 +600,7 @@ def main(argsl=None, # type: Optional[List[str]]
_logger.error("--provenance incompatible with --no-compute-checksum")
return 1
ro = ResearchObject(
getdefault(runtimeContext.make_fs_access, StdFsAccess),
temp_prefix_ro=args.tmpdir_prefix, orcid=args.orcid,
full_name=args.cwl_full_name)
runtimeContext.research_obj = ro
......@@ -795,7 +798,8 @@ def main(argsl=None, # type: Optional[List[str]]
relative_deps=args.relative_deps,
make_fs_access=runtimeContext.make_fs_access,
input_basedir=input_basedir,
secret_store=runtimeContext.secret_store)
secret_store=runtimeContext.secret_store,
input_required=input_required)
except SystemExit as err:
return err.code
......@@ -833,6 +837,21 @@ def main(argsl=None, # type: Optional[List[str]]
if runtimeContext.research_obj is not None:
runtimeContext.research_obj.create_job(
out, None, True)
def remove_at_id(doc): # type: (MutableMapping[Text, Any]) -> None
for key in list(doc.keys()):
if key == '@id':
del doc[key]
else:
value = doc[key]
if isinstance(value, MutableMapping):
remove_at_id(value)
elif isinstance(value, MutableSequence):
for entry in value:
if isinstance(entry, MutableMapping):
remove_at_id(entry)
remove_at_id(out)
visit_class(out, ("File",), functools.partial(
add_sizes, runtimeContext.make_fs_access('')))
def loc_to_path(obj): # type: (Dict[Text, Any]) -> None
for field in ("path", "nameext", "nameroot", "dirname"):
......
cwltool/provenance.py
View file @ b5a53e1c
......@@ -305,9 +305,11 @@ class ProvenanceProfile():
host_provenance, # type: bool
user_provenance, # type: bool
orcid, # type: str
fsaccess, # type: StdFsAccess
run_uuid=None # type: Optional[uuid.UUID]
): # type: (...) -> None
"""Initialize the provenance profile."""
self.fsaccess = fsaccess
self.orcid = orcid
self.research_object = research_object
self.folder = self.research_object.folder
......@@ -504,8 +506,7 @@ class ProvenanceProfile():
if not entity and 'location' in value:
location = str(value['location'])
# If we made it here, we'll have to add it to the RO
fsaccess = StdFsAccess("")
with fsaccess.open(location, "rb") as fhandle:
with self.fsaccess.open(location, "rb") as fhandle:
relative_path = self.research_object.add_data_file(fhandle)
# FIXME: This naively relies on add_data_file setting hash as filename
checksum = PurePath(relative_path).name
......@@ -545,7 +546,12 @@ class ProvenanceProfile():
# Check for secondaries
for sec in value.get("secondaryFiles", ()):
# TODO: Record these in a specializationOf entity with UUID?
if sec['class'] == "File":
(sec_entity, _, _) = self.declare_file(sec)
elif sec['class'] == "Directory":
sec_entity = self.declare_directory(sec)
else:
raise ValueError("Got unexpected secondaryFiles value: {}".format(sec))
# We don't know how/when/where the secondary file was generated,
# but CWL convention is a kind of summary/index derived
# from the original file. As its generally in a different format
......@@ -592,8 +598,7 @@ class ProvenanceProfile():
is_empty = True
if "listing" not in value:
fsaccess = StdFsAccess("")
get_listing(fsaccess, value)
get_listing(self.fsaccess, value)
for entry in value.get("listing", []):
is_empty = False
# Declare child-artifacts
......@@ -796,10 +801,13 @@ class ProvenanceProfile():
base += "/" + name
for key, value in job_order.items():
prov_role = self.wf_ns["%s/%s" % (base, key)]
try:
entity = self.declare_artefact(value)
self.document.used(
process_run_id, entity, datetime.datetime.now(), None,
{"prov:role": prov_role})
except OSError:
pass
def generate_output_prov(self,
final_output, # type: Union[Dict[Text, Any], List[Dict[Text, Any]]]
......@@ -937,8 +945,8 @@ class ProvenanceProfile():
class ResearchObject():
"""CWLProv Research Object."""
def __init__(self, temp_prefix_ro="tmp", orcid='', full_name=''):
# type: (str, Text, Text) -> None
def __init__(self, fsaccess, temp_prefix_ro="tmp", orcid='', full_name=''):
# type: (StdFsAccess, str, Text, Text) -> None
"""Initialize the ResearchObject."""
self.temp_prefix = temp_prefix_ro
self.orcid = '' if not orcid else _valid_orcid(orcid)
......@@ -954,7 +962,7 @@ class ResearchObject():
self._external_aggregates = [] # type: List[Dict[Text, Text]]
self.annotations = [] # type: List[Dict[Text, Any]]
self._content_types = {} # type: Dict[Text,str]
self.fsaccess = fsaccess
# These should be replaced by generate_prov_doc when workflow/run IDs are known:
self.engine_uuid = "urn:uuid:%s" % uuid.uuid4()
self.ro_uuid = uuid.uuid4()
......@@ -1553,8 +1561,7 @@ class ResearchObject():
# Register in RO; but why was this not picked
# up by used_artefacts?
_logger.info("[provenance] Adding to RO %s", structure["location"])
fsaccess = StdFsAccess("")
with fsaccess.open(structure["location"], "rb") as fp:
with self.fsaccess.open(structure["location"], "rb") as fp:
relative_path = self.add_data_file(fp)
checksum = PurePosixPath(relative_path).name
structure["checksum"] = "%s$%s" % (SHA1, checksum)
......@@ -1572,7 +1579,10 @@ class ResearchObject():
del structure["location"]
for val in structure.values():
try:
self._relativise_files(val)
except OSError:
pass
return
if isinstance(structure, (str, Text)):
......
cwltool/singularity.py
View file @ b5a53e1c
......@@ -118,6 +118,8 @@ class SingularityCommandLineJob(ContainerCommandLineJob):
if not match:
dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
elif "dockerImageId" in dockerRequirement:
if os.path.isfile(dockerRequirement['dockerImageId']):
found = True
candidates.append(dockerRequirement['dockerImageId'])
candidates.append(_normalize_image_id(dockerRequirement['dockerImageId']))
if is_version_3_or_newer():
......
cwltool/workflow.py
View file @ b5a53e1c
......@@ -522,7 +522,8 @@ class Workflow(Process):
host_provenance=loadingContext.host_provenance,
user_provenance=loadingContext.user_provenance,
orcid=loadingContext.orcid,
run_uuid=run_uuid) # inherit RO UUID for master wf run
run_uuid=run_uuid,
fsaccess=loadingContext.research_obj.fsaccess) # inherit RO UUID for master wf run
# TODO: Is Workflow(..) only called when we are the master workflow?
self.parent_wf = self.provenance_object
......@@ -582,6 +583,7 @@ class Workflow(Process):
if runtimeContext.research_obj is not None:
if runtimeContext.toplevel:
# Record primary-job.json
runtimeContext.research_obj.fsaccess = runtimeContext.make_fs_access('')
runtimeContext.research_obj.create_job(builder.job, self.job)
job = WorkflowJob(self, runtimeContext)
......
debian/changelog
View file @ b5a53e1c
cwltool (1.0.20190915164430+dfsg-1) unstable; urgency=medium
* New upstream version
-- Michael R. Crusoe <michael.crusoe@gmail.com> Sun, 15 Sep 2019 21:17:19 +0200
cwltool (1.0.20190815141648+dfsg-2) unstable; urgency=medium
* Fix AutoPkgTest requirements (Closes: #935579)
......
debian/control
View file @ b5a53e1c
......@@ -5,6 +5,7 @@ Section: science
Priority: optional
Build-Depends: debhelper-compat (= 12),
dh-python,
procps,
python3,
python3-all,
python3-setuptools,
......
debian/patches/disable_prov
View file @ b5a53e1c
......@@ -75,10 +75,11 @@ Description: Disable --provance until python3-proc issue is resolved
# Module to be tested
from cwltool import load_tool, provenance
from cwltool.main import main
@@ -48,635 +46,4 @@
@@ -48,636 +46,3 @@
if not os.environ.get("DEBUG"):
shutil.rmtree(directory)
-
-
-def cwltool(folder, *args):
- new_args = ['--provenance', folder]
......@@ -544,7 +545,7 @@ Description: Disable --provance until python3-proc issue is resolved
-
-@pytest.fixture
-def research_object():
- re_ob = provenance.ResearchObject()
- re_ob = provenance.ResearchObject(StdFsAccess(''))
- yield re_ob
- re_ob.close()
-
......@@ -711,4 +712,3 @@ Description: Disable --provance until python3-proc issue is resolved
-# Reasearch object may need to be pickled (for Toil)
-def test_research_object_picklability(research_object):
- assert pickle.dumps(research_object) is not None
+# For now, we do not ship the --provenance feature in Debian
debian/patches/pytest
View file @ b5a53e1c
......@@ -11,7 +11,7 @@ Description: Deal with pytest 4.6 changes
processes_to_kill, versionstring, visit_class,
windows_default_container_id)
from .subgraph import get_subgraph
@@ -416,7 +416,7 @@
@@ -417,7 +417,7 @@
base = os.getcwd()
visit_class(deps, ("File", "Directory"), functools.partial(
make_relative, base))
......@@ -20,8 +20,8 @@ Description: Deal with pytest 4.6 changes
def prov_deps(obj, # type: Mapping[Text, Any]
document_loader, # type: Loader
@@ -493,8 +493,8 @@
runtimeContext=None # type: Optional[RuntimeContext]
@@ -495,8 +495,8 @@
input_required=True # type: bool
): # type: (...) -> int
if not stdout: # force UTF-8 even if the console is configured differently
- if (hasattr(sys.stdout, "encoding")
......@@ -31,7 +31,7 @@ Description: Deal with pytest 4.6 changes
if PY3 and hasattr(sys.stdout, "detach"):
stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
else:
@@ -681,7 +681,7 @@
@@ -684,7 +684,7 @@
print_pack(loadingContext.loader, processobj, uri, metadata))
if args.print_pre:
......@@ -40,7 +40,7 @@ Description: Deal with pytest 4.6 changes
return 0
tool = make_tool(uri, loadingContext)
@@ -737,7 +737,7 @@
@@ -740,7 +740,7 @@
if args.print_subgraph:
if "name" in tool.tool:
del tool.tool["name"]
......@@ -49,7 +49,7 @@ Description: Deal with pytest 4.6 changes
return 0
except (validate.ValidationException) as exc:
@@ -849,7 +849,7 @@
@@ -868,7 +868,7 @@
if isinstance(out, string_types):
stdout.write(out)
else:
......@@ -71,7 +71,7 @@ Description: Deal with pytest 4.6 changes
--- cwltool.orig/tests/test_udocker.py
+++ cwltool/tests/test_udocker.py
@@ -37,12 +37,9 @@
@@ -38,12 +38,9 @@
break
subprocess.call(["rm", "./udocker"])
......
debian/tests/control
View file @ b5a53e1c
Tests: run-tests
Depends: @, python3-pytest-xdist, python3-pytest-mock, python3-pytest-runner, nodejs, curl, python3-mock
Depends: @, python3-pytest-xdist, python3-pytest-mock, python3-pytest-runner, nodejs, curl, python3-mock, procps
Restrictions: allow-stderr
debian/tests/run-tests
View file @ b5a53e1c
......@@ -6,6 +6,7 @@ if [ "$ADTTMP" = "" ] ; then
fi
cd "$ADTTMP"
export HOME=${ADTTMP}
SKIPS=
# if [ "$(id -u)" -eq 0 ]; then
......
setup.cfg
View file @ b5a53e1c
......@@ -13,6 +13,6 @@ addopts = --ignore cwltool/schemas --basetemp ./tmp
testpaths = tests
[egg_info]
tag_build = .20190815141648
tag_build = .20190915164430
tag_date = 0