Skip to content
Commits on Source (7)
# CHANGELOG
## 1.7.0
- expose `SmrtLinkAuthClient` to `pbcommand.services`
- mark `ServiceAccessLayer` as deprecated. For SL >= 6.X.X, unauth'ed access is limited to internal usecases, such as `pbtestkit-service-runner` and `pbsmrtpipe`.
\ No newline at end of file
include *.txt *.md
recursive-include examples *.txt *.py
include pbcommand/schemas/*.avsc
.PHONY: all clean install dev-install test doc
SHELL = /bin/bash -e
WHEELHOUSE?=./wheelhouse
all: install
......@@ -22,8 +23,10 @@ clean:
test-nose:
nosetests -s --verbose --with-xunit --logging-config log_nose.cfg tests/test_*.py
test-pytest:
pytest -v --durations=12 --junitxml=nosetests.xml --cov=./pbcommand --cov-report=xml:coverage.xml tests/test_*.py
test: test-nose run-pylint run-pep8
test: test-pytest run-pylint run-pep8
doc:
cd docs && make html
......@@ -52,7 +55,7 @@ build-java-classes:
extract-readme-snippets:
rm -rf readme-snippet-*.py
pandoc -t markdown README.md | pandoc --filter ./extract-readme-snippets.py
pandoc -t markdown README.md | pandoc --filter ./bin/extract-readme-snippets.py
build-avro-schema-docs:
# this requires nodejs + https://github.com/ept/avrodoc
......@@ -60,3 +63,8 @@ build-avro-schema-docs:
cpp-header:
python extras/make_cpp_file_types_header.py FileTypes.h
wheel:
which pip
pip wheel -v --wheel-dir=${WHEELHOUSE} --no-deps .
ls -larth ${WHEELHOUSE}
avro
requests
iso8601
pytz
nose
coverage
tox
autopep8
pylint
# Putting these here for RTD
sphinx-argparse
sphinx-bootstrap-theme
# I think this is a bug in jupyter client
# for some reason jupyter_client.manager is called
# which generates an ImportError
# maybe related https://github.com/jupyter/nbconvert/issues/495
jupyter_client
# for ipython notebooks in sphinx docs
nbsphinx
avro
requests
iso8601
# For sphinx extension
ipython
# ipython requires this?
matplotlib
#!/bin/bash -ex
#!/bin/bash
NX3PBASEURL=http://nexus/repository/unsupported/pitchfork/gcc-4.9.2
export PATH=$PWD/bin:/mnt/software/a/anaconda2/4.2.0/bin:$PATH
export PYTHONUSERBASE=$PWD
export CFLAGS="-I/mnt/software/a/anaconda2/4.2.0/include"
PIP="pip --cache-dir=$bamboo_build_working_directory/.pip"
type module >& /dev/null || . /mnt/software/Modules/current/init/bash
module load gcc/4.9.2
rm -rf bin lib include share
mkdir bin lib include share
$PIP install --user \
iso8601
$PIP install --user \
$NX3PBASEURL/pythonpkgs/xmlbuilder-1.0-cp27-none-any.whl \
$NX3PBASEURL/pythonpkgs/tabulate-0.7.5-cp27-none-any.whl \
$NX3PBASEURL/pythonpkgs/pysam-0.9.1.4-cp27-cp27mu-linux_x86_64.whl \
$NX3PBASEURL/pythonpkgs/avro-1.7.7-cp27-none-any.whl
$PIP install --user --upgrade pylint
$PIP install --user -r REQUIREMENTS.txt
$PIP install --user -r REQUIREMENTS_TEST.txt
$PIP install --user -e ./
nosetests -s --verbose --with-xunit --xunit-file=nosetests.xml --with-coverage --cover-xml --cover-xml-file=coverage.xml --logging-config \
log_nose.cfg tests/test_*.py
sed -i -e 's@filename="@filename="./@g' coverage.xml
make run-pylint run-pep8
module purge
module load gcc
module load ccache
module load python/3
module load htslib # since pysam was built against this
#module load make
set -vex
which gcc
which g++
gcc --version
which python
python --version
export PYTHONUSERBASE=$(pwd)/LOCAL
export PATH=${PYTHONUSERBASE}/bin:${PATH}
PIP="pip --cache-dir=$bamboo_build_working_directory/.pip"
if [[ -z ${bamboo_repository_branch_name+x} ]]; then
WHEELHOUSE=/mnt/software/p/python/wheelhouse/develop
elif [[ ${bamboo_repository_branch_name} == develop ]]; then
WHEELHOUSE=/mnt/software/p/python/wheelhouse/develop
elif [[ ${bamboo_repository_branch_name} == master ]]; then
WHEELHOUSE=/mnt/software/p/python/wheelhouse/master
else
WHEELHOUSE=/mnt/software/p/python/wheelhouse/develop
fi
export WHEELHOUSE
rm -rf build
mkdir -p build/{bin,lib,include,share}
# Delete avro.
# avro-python3 would work, but avro must be deleted no matter way.
# https://stackoverflow.com/questions/40732419/how-to-read-avro-files-in-python-3-5-2
rm -rf ${PYTHONUSERBASE}/lib/python3*/site-packages/avro*
pip install --user --no-index --find-link "${WHEELHOUSE}" --no-compile -e '.[test]'
make test
##########################################################################
bash bamboo_wheel.sh
#!/bin/bash
type module >& /dev/null || . /mnt/software/Modules/current/init/bash
module purge
module load gcc
module load ccache
set -vex
ls -larth ..
ls -larth
pwd
case "${bamboo_planRepository_branchName}" in
develop|master)
echo "Building wheels ..."
;;
*)
echo "Not building wheels."
exit
;;
esac
export WHEELHOUSE=./wheelhouse
# Give everybody read/write access.
umask 0000
module load python/3
make wheel
# http://bamboo.pacificbiosciences.com:8085/build/admin/edit/defaultBuildArtifact.action?buildKey=SAT-TAGDEPS-JOB1
# For old artifact config:
#mkdir -p ./artifacts/gcc-6.4.0/wheelhouse
#rsync -av ${WHEELHOUSE}/falcon*.whl artifacts/gcc-6.4.0/wheelhouse/
# Select export dir based on Bamboo branch, but only for develop and master.
case "${bamboo_planRepository_branchName}" in
develop|master)
WHEELHOUSE="/mnt/software/p/python/wheelhouse/${bamboo_planRepository_branchName}/"
rsync -av ./wheelhouse/ ${WHEELHOUSE}
;;
*)
;;
esac
python-pbcommand (1.1.1+git20191122.ec024c3-1) unstable; urgency=medium
* New upstream version Git checkout
Drop Python2 support and switch to Python3
Closes: #932543
* debhelper-compat 12
* Standards-Version: 4.4.1
TODO: python3-xmlbuilder (#938277)
-- Andreas Tille <tille@debian.org> Sat, 07 Dec 2019 21:44:15 +0100
python-pbcommand (1.1.1-1) unstable; urgency=medium
* Team upload.
......
......@@ -2,36 +2,35 @@ Source: python-pbcommand
Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
Section: python
Priority: optional
Build-Depends: debhelper (>= 11~),
Build-Depends: debhelper-compat (= 12),
dh-python,
python-all,
python-setuptools,
python-xmlbuilder,
python-jsonschema,
python-avro,
python-numpy,
python-requests,
python-iso8601,
# For tests
python-nose,
python-pbcore
Standards-Version: 4.2.1
python3-all,
python3-setuptools,
python3-xmlbuilder,
python3-jsonschema,
python3-avro,
python3-numpy,
python3-requests,
python3-iso8601,
python3-nose <!nocheck>,
python3-pbcore <!nocheck>
Standards-Version: 4.4.1
Vcs-Browser: https://salsa.debian.org/med-team/python-pbcommand
Vcs-Git: https://salsa.debian.org/med-team/python-pbcommand.git
Homepage: https://pbcommand.readthedocs.org/en/latest/
Package: python-pbcommand
Package: python3-pbcommand
Architecture: all
Depends: ${misc:Depends},
${python:Depends},
python-numpy
Recommends: python-pbcore
${python3:Depends},
python3-numpy
Recommends: python3-pbcore
Description: common command-line interface for Pacific Biosciences analysis modules
To integrate with the pbsmrtpipe workflow engine, one must to be able to
generate a Tool Contract and to be able to run from a Resolved Tool Contract.
A Tool Contract contains the metadata of the exe, such as the file types of
inputs, outputs and options.
There are two principal use cases, first wrapping/calling Python functions that
have been defined in external Python packages, or scripts. Second, creating a
have been defined in external Python 3 packages, or scripts. Second, creating a
CLI tool that supports emitting tool contracts, running resolved tool contracts
and complete argparse-style CLI.
......@@ -8,7 +8,7 @@ include /usr/share/dpkg/default.mk
export PYBUILD_NAME=pbcommand
%:
LC_ALL=C.UTF-8 dh $@ --with python2 --buildsystem=pybuild
LC_ALL=C.UTF-8 dh $@ --with python3 --buildsystem=pybuild
override_dh_auto_test:
ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
......
version=3
version=4
opts="filenamemangle=s/(?:.*?)?v?(\d[\d.]*)\.tar\.gz/python-pbcommand-$1.tar.gz/" \
https://github.com/PacificBiosciences/pbcommand/releases (?:.*?/)?v?(\d[\d.]*)\.tar\.gz
#opts="filenamemangle=s/(?:.*?)?v?(\d[\d.]*)\.tar\.gz/python-pbcommand-$1.tar.gz/" \
# https://github.com/PacificBiosciences/pbcommand/releases (?:.*?/)?v?(\d[\d.]*)\.tar\.gz
# Upstream is not tagging releases regularly - thus we use mode=git
# Upstream is now actually tagging releases - but there is no official Python3 release yet
opts="mode=git,pretty=1.1.1+git%cd.%h,repacksuffix=+dfsg,dversionmangle=auto,repack,compression=xz" \
https://github.com/PacificBiosciences/pbcommand.git HEAD
import pprint
from pbcommand.models.common import (FileTypes, ResourceTypes, SymbolTypes, TaskTypes)
from pbcommand.models.common import (
FileTypes, ResourceTypes, SymbolTypes, TaskTypes)
from pbcommand.models.parser import get_pbparser
from pbcommand.models.tool_contract import ToolDriver
def _example_options(p):
p.add_input_file_type(FileTypes.BAM, "ubam", "Unaligned BAM", "A General description of BAM")
p.add_input_file_type(FileTypes.DS_REF, "ref", "Reference", "Reference Dataset XML")
p.add_int("mytool.task_options.myoption", "myopt", 7, "My Option", "My Option which does this and that")
p.add_str("mytool.task_options.myopt2", "mylabel", "Stuff", "My Option name", "My Option2 description")
p.add_output_file_type(FileTypes.REPORT, "rpt", "Json Report", "Mapping Stats Report Task", "mapping-stats.report.json")
p.add_input_file_type(
FileTypes.BAM,
"ubam",
"Unaligned BAM",
"A General description of BAM")
p.add_input_file_type(
FileTypes.DS_REF,
"ref",
"Reference",
"Reference Dataset XML")
p.add_int("mytool.task_options.myoption", "myopt", 7,
"My Option", "My Option which does this and that")
p.add_str(
"mytool.task_options.myopt2",
"mylabel",
"Stuff",
"My Option name",
"My Option2 description")
p.add_output_file_type(
FileTypes.REPORT,
"rpt",
"Json Report",
"Mapping Stats Report Task",
"mapping-stats.report.json")
return p
def example_01():
driver = ToolDriver("my-exe --config")
resource_types = (ResourceTypes.TMP_DIR, ResourceTypes.LOG_FILE)
p = get_pbparser("pbcommand.tools.example", "0.1.2", "My Description", driver, TaskTypes.DISTRIBUTED, SymbolTypes.MAX_NPROC, resource_types)
p = get_pbparser(
"pbcommand.tools.example",
"0.1.2",
"My Description",
driver,
TaskTypes.DISTRIBUTED,
SymbolTypes.MAX_NPROC,
resource_types)
return _example_options(p)
def example_02():
p = example_01()
print "Generated Manifest"
print("Generated Manifest")
pprint.pprint(p.parsers[1].to_tool_contract())
# ipython will dump out here. with non-zero exitcode. blah...
print "Running Argparse --help"
print("Running Argparse --help")
p.parsers[0].parser.parse_args(["--help"])
return p
......@@ -18,8 +18,10 @@ def run(argv):
for ft_name in dir(FileTypes):
ft = getattr(FileTypes, ft_name)
if isinstance(ft, FileType):
out.write("""#define {f} "{i}"\n""".format(f=ft_name, i=ft.file_type_id))
print "Wrote {f}".format(f=args.out)
out.write(
"""#define {f} "{i}"\n""".format(
f=ft_name, i=ft.file_type_id))
print("Wrote {f}".format(f=args.out))
return 0
......
[loggers]
keys=root,log01
[logger_root]
#level=DEBUG
level=NOTSET
handlers=hand01
[logger_log01]
level=NOTSET
handlers=hand01
propagate=1
qualname=""
[handlers]
keys=hand01
[filters]
[formatters]
keys=form01
[handler_hand01]
class=FileHandler
level=DEBUG
formatter=form01
args=('reports_unittests.log', 'w')
[formatter_form01]
format=[%(levelname)s] %(asctime)-15s [%(name)s %(funcName)s %(lineno)d] %(message)s
datefmt=
class=logging.Formatter
VERSION = (1, 1, 1)
import pkg_resources
import sys
try:
__VERSION__ = pkg_resources.get_distribution('pbcommand').version
except Exception:
__VERSION__ = 'unknown'
VERSION = (int(x) for x in __VERSION__.split('.'))
def get_version():
......@@ -9,3 +17,11 @@ def get_version():
.. note:: This should be improved to be compliant with PEP 386.
"""
return ".".join([str(i) for i in VERSION])
def to_ascii(s):
return s
def to_utf8(s):
return s
from .core import (pacbio_args_runner,
pacbio_args_or_contract_runner,
pbparser_runner,
get_default_argparser, get_default_argparser_with_base_opts)
from .quick import (registry_runner, registry_builder, QuickOpt)
""" New Commandline interface that supports ResolvedToolContracts and emitting ToolContracts
"""
New Commandline interface that supports ResolvedToolContracts and emitting ToolContracts
There's three use cases.
......@@ -11,29 +12,23 @@ Going to do this in a new steps.
- de-serializing of RTC (I believe this should be done via avro, not a new random JSON file. With avro, the java, c++, classes can be generated. Python can load the RTC via a structure dict that has a well defined schema)
- get loading and running of RTC from commandline to call main func in a report.
- generate/emit TC from a a common commandline parser interface that builds the TC and the standard argparse instance
"""
from __future__ import print_function
import argparse
import errno
import json
import logging
import time
import traceback
import shutil
import os
import shutil
import sys
import errno
import time
import traceback
import pbcommand
from pbcommand.models import PbParser, ResourceTypes
from pbcommand.common_options import (RESOLVED_TOOL_CONTRACT_OPTION,
EMIT_TOOL_CONTRACT_OPTION,
add_resolved_tool_contract_option,
add_base_options)
from pbcommand.models import ResourceTypes, PacBioAlarm
from pbcommand.models.report import Report, Attribute
from pbcommand.common_options import add_base_options, add_nproc_option
from pbcommand.utils import get_parsed_args_log_level
from pbcommand.pb_io.tool_contract_io import load_resolved_tool_contract_from
def _add_version(p, version):
......@@ -62,7 +57,8 @@ def get_default_argparser(version, description):
return _add_version(p, version)
def get_default_argparser_with_base_opts(version, description, default_level="INFO"):
def get_default_argparser_with_base_opts(
version, description, default_level="INFO", nproc=None):
"""Return a parser with the default log related options
If you don't want the default log behavior to go to stdout, then set
......@@ -84,7 +80,29 @@ def get_default_argparser_with_base_opts(version, description, default_level="IN
my-tool --my-opt=1234 --log-level=DEBUG --log-file=file.log file_in.txt
"""
return add_base_options(get_default_argparser(version, description), default_level=default_level)
p = add_base_options(
get_default_argparser(
version,
description),
default_level=default_level)
if nproc is not None:
p = add_nproc_option(p)
return p
def write_task_report(run_time, nproc, exit_code):
attributes = [
Attribute("host", value=os.uname()[1]),
Attribute("system", value=os.uname()[0]),
Attribute("nproc", value=nproc),
Attribute("run_time", value=run_time),
Attribute("exit_code", value=exit_code)
]
report = Report("workflow_task",
title="Workflow Task Report",
attributes=attributes,
tags=("internal",))
report.write_json("task-report.json")
def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
......@@ -118,9 +136,21 @@ def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
log_file = getattr(pargs, 'log_file', None)
# Currently, only support to stdout. More customization would require
# more required commandline options in base parser (e.g., --log-file, --log-formatter)
# more required commandline options in base parser (e.g., --log-file,
# --log-formatter)
log_options = dict(level=level, file_name=log_file)
base_dir = os.getcwd()
dump_alarm_on_error = False
if "dump_alarm_on_error" in kwargs:
dump_alarm_on_error = kwargs.pop("dump_alarm_on_error")
is_cromwell_environment = bool(
os.environ.get(
"SMRT_PIPELINE_BUNDLE_DIR",
None)) and "cromwell-executions" in base_dir
dump_alarm_on_error = dump_alarm_on_error and is_cromwell_environment
# The Setup log func must adhere to the pbcommand.utils.setup_log func
# signature
# FIXME. This should use the more concrete F(file_name_or_name, level, formatter)
......@@ -128,9 +158,16 @@ def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
if setup_log_func is not None and alog is not None:
setup_log_func(alog, **log_options)
alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version()))
alog.info("completed setting up logger with {f}".format(f=setup_log_func))
alog.info(
"completed setting up logger with {f}".format(
f=setup_log_func))
alog.info("log opts {d}".format(d=log_options))
if dump_alarm_on_error:
alog.info(
"This command appears to be running as part of a Cromwell workflow")
alog.info("Additional output files may be generated")
try:
# the code in func should catch any exceptions. The try/catch
# here is a fail safe to make sure the program doesn't fail
......@@ -143,6 +180,14 @@ def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
alog.error(e, exc_info=True)
else:
traceback.print_exc(sys.stderr)
if dump_alarm_on_error:
PacBioAlarm.dump_error(
file_name=os.path.join(base_dir, "alarms.json"),
exception=e,
info="".join(traceback.format_exc()),
message=str(e),
name=e.__class__.__name__,
severity=logging.ERROR)
# We should have a standard map of exit codes to Int
if isinstance(e, IOError):
......@@ -151,121 +196,17 @@ def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
return_code = 2
_d = dict(r=return_code, s=run_time)
if is_cromwell_environment:
alog.info("Writing task report to task-report.json")
write_task_report(run_time, getattr(pargs, "nproc", 1), return_code)
if alog is not None:
alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d))
return return_code
def pacbio_args_runner(argv, parser, args_runner_func, alog, setup_log_func):
def pacbio_args_runner(argv, parser, args_runner_func, alog, setup_log_func,
dump_alarm_on_error=True):
# For tools that haven't yet implemented the ToolContract API
args = parser.parse_args(argv)
return _pacbio_main_runner(alog, setup_log_func, args_runner_func, args)
class TemporaryResourcesManager(object):
"""Context manager for creating and destroying temporary resources"""
def __init__(self, rtc):
self.resolved_tool_contract = rtc
def __enter__(self):
for resource in self.resolved_tool_contract.task.resources:
if resource.type_id == ResourceTypes.TMP_DIR or resource.type_id == ResourceTypes.TMP_FILE:
try:
dirname = os.path.dirname(os.path.realpath(resource.path))
os.makedirs(dirname)
except EnvironmentError as e:
# IOError, OSError subclass Environment and add errno
# Note, dirname is not strictly defined here. If there's an
# Env exception caught from the right hand this will raise NameError
# and will still fail, but with a different exception type.
if e.errno == errno.EEXIST and os.path.isdir(dirname):
pass
else:
raise
if resource.type_id == ResourceTypes.TMP_DIR:
os.makedirs(resource.path)
def __exit__(self, type, value, traceback):
for resource in self.resolved_tool_contract.task.resources:
if resource.type_id == ResourceTypes.TMP_DIR:
if os.path.exists(resource.path):
shutil.rmtree(resource.path)
def pacbio_args_or_contract_runner(argv,
parser,
args_runner_func,
contract_tool_runner_func,
alog, setup_log_func):
"""
For tools that understand resolved_tool_contracts, but can't emit
tool contracts (they may have been written by hand)
:param parser: argparse Parser
:type parser: ArgumentParser
:param args_runner_func: func(args) => int signature
:param contract_tool_runner_func: func(tool_contract_instance) should be the signature
:param alog: a python log instance
:param setup_log_func: func(log_instance) => void signature
:return: int return code
:rtype: int
"""
def _log_not_none(msg):
if alog is not None:
alog.info(msg)
# circumvent the argparse parsing by inspecting the raw argv, then create
# a temporary parser with limited arguments to process the special case of
# --resolved-cool-contract (while still respecting verbosity flags).
if any(a.startswith(RESOLVED_TOOL_CONTRACT_OPTION) for a in argv):
p_tmp = get_default_argparser(version=parser.version,
description=parser.description)
add_resolved_tool_contract_option(add_base_options(p_tmp,
default_level="NOTSET"))
args_tmp = p_tmp.parse_args(argv)
resolved_tool_contract = load_resolved_tool_contract_from(
args_tmp.resolved_tool_contract)
_log_not_none("Successfully loaded resolved tool contract from {a}".format(a=argv))
# XXX if one of the logging flags was specified, that takes precedence,
# otherwise use the log level in the resolved tool contract. note that
# this takes advantage of the fact that argparse allows us to use
# NOTSET as the default level even though it's not one of the choices.
log_level = get_parsed_args_log_level(args_tmp,
default_level=logging.NOTSET)
if log_level == logging.NOTSET:
log_level = resolved_tool_contract.task.log_level
with TemporaryResourcesManager(resolved_tool_contract) as tmp_mgr:
r = _pacbio_main_runner(alog, setup_log_func, contract_tool_runner_func, resolved_tool_contract, level=log_level)
_log_not_none("Completed running resolved contract. {c}".format(c=resolved_tool_contract))
return r
else:
# tool was called with the standard commandline invocation
return pacbio_args_runner(argv, parser, args_runner_func, alog,
setup_log_func)
def pbparser_runner(argv,
parser,
args_runner_func,
contract_runner_func,
alog,
setup_log_func):
"""Run a Contract or emit a contract to stdout."""
if not isinstance(parser, PbParser):
raise TypeError("Only supports PbParser.")
arg_parser = parser.arg_parser.parser
# extract the contract
tool_contract = parser.to_contract()
if EMIT_TOOL_CONTRACT_OPTION in argv:
# print tool_contract
x = json.dumps(tool_contract.to_dict(), indent=4, separators=(',', ': '))
print(x)
else:
return pacbio_args_or_contract_runner(argv, arg_parser, args_runner_func, contract_runner_func, alog, setup_log_func)
return _pacbio_main_runner(alog, setup_log_func, args_runner_func, args,
dump_alarm_on_error=dump_alarm_on_error)