Makefile

@@ -17,11 +17,14 @@ clean:
 	find . -name "*.err" | xargs rm -f;\
 	find . -name "*.log" | xargs rm -f;\
 	rm -rf dist;\
-	rm -rf docs/_build
+	rm -rf docs/build
+	rm -f FileTypes.h
 
-test:
+test-nose:
 	nosetests -s --verbose --with-xunit --logging-config log_nose.cfg tests/test_*.py
 
+test: test-nose run-pylint run-pep8
+
 doc:
 	cd docs && make html
 
@@ -34,8 +37,12 @@ build-tool-contracts:
 	python -m pbcommand.cli.examples.dev_scatter_fasta_app --emit-tool-contract > ./tests/data/tool-contracts/dev_scatter_fasta_app_tool_contract.json
 	python -m pbcommand.cli.examples.dev_quick_hello_world emit-tool-contracts -o ./tests/data/tool-contracts
 
+run-pylint:
+	pylint --errors-only pbcommand
+
 run-pep8:
-	find pbcommand -name "*.py" -exec pep8 --ignore=E501,E265,E731,E402,W292 {} \;
+	# use xargs to propagate exit code
+	find pbcommand -name "*.py" | xargs pep8 --ignore=E501,E265,E731,E402,W292
 
 run-auto-pep8:
 	find pbcommand -name "*.py" -exec autopep8 -i --ignore=E501,E265,E731,E402,W292 {} \;
@@ -50,3 +57,6 @@ extract-readme-snippets:
 build-avro-schema-docs:
 	# this requires nodejs + https://github.com/ept/avrodoc
 	avrodoc pbcommand/schemas/*.avsc > index.html
+
+cpp-header:
+	python extras/make_cpp_file_types_header.py FileTypes.h

README.md

@@ -310,3 +310,7 @@ optional arguments:
                         Min Sequence Length filter (default: 25)
 ```
 
+
+DISCLAIMER
+----------
+THIS WEBSITE AND CONTENT AND ALL SITE-RELATED SERVICES, INCLUDING ANY DATA, ARE PROVIDED "AS IS," WITH ALL FAULTS, WITH NO REPRESENTATIONS OR WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, SATISFACTORY QUALITY, NON-INFRINGEMENT OR FITNESS FOR A PARTICULAR PURPOSE. YOU ASSUME TOTAL RESPONSIBILITY AND RISK FOR YOUR USE OF THIS SITE, ALL SITE-RELATED SERVICES, AND ANY THIRD PARTY WEBSITES OR APPLICATIONS. NO ORAL OR WRITTEN INFORMATION OR ADVICE SHALL CREATE A WARRANTY OF ANY KIND. ANY REFERENCES TO SPECIFIC PRODUCTS OR SERVICES ON THE WEBSITES DO NOT CONSTITUTE OR IMPLY A RECOMMENDATION OR ENDORSEMENT BY PACIFIC BIOSCIENCES.

REQUIREMENTS.txt

 avro
 requests
 iso8601
+pytz

REQUIREMENTS_TEST.txt

 nose
+coverage
 tox
+autopep8
+pylint
 # Putting these here for RTD
 sphinx-argparse
 sphinx-bootstrap-theme
+# I think this is a bug in jupyter client
+# for some reason jupyter_client.manager is called
+# which generates an ImportError
+# maybe related https://github.com/jupyter/nbconvert/issues/495
+jupyter_client
+# for ipython notebooks in sphinx docs
+nbsphinx
 avro
 requests
 iso8601

bamboo_build.sh

+#!/bin/bash -ex
+
+NX3PBASEURL=http://nexus/repository/unsupported/pitchfork/gcc-4.9.2
+export PATH=$PWD/bin:/mnt/software/a/anaconda2/4.2.0/bin:$PATH
+export PYTHONUSERBASE=$PWD
+export CFLAGS="-I/mnt/software/a/anaconda2/4.2.0/include"
+PIP="pip --cache-dir=$bamboo_build_working_directory/.pip"
+type module >& /dev/null || . /mnt/software/Modules/current/init/bash
+module load gcc/4.9.2
+
+rm -rf bin lib include share
+mkdir  bin lib include share
+
+$PIP install --user \
+  iso8601
+$PIP install --user \
+  $NX3PBASEURL/pythonpkgs/xmlbuilder-1.0-cp27-none-any.whl \
+  $NX3PBASEURL/pythonpkgs/tabulate-0.7.5-cp27-none-any.whl \
+  $NX3PBASEURL/pythonpkgs/pysam-0.9.1.4-cp27-cp27mu-linux_x86_64.whl \
+  $NX3PBASEURL/pythonpkgs/avro-1.7.7-cp27-none-any.whl
+
+$PIP install --user --upgrade pylint
+$PIP install --user -r  REQUIREMENTS.txt
+$PIP install --user -r  REQUIREMENTS_TEST.txt
+$PIP install --user -e ./
+nosetests -s --verbose --with-xunit --xunit-file=nosetests.xml --with-coverage --cover-xml --cover-xml-file=coverage.xml --logging-config \
+    log_nose.cfg tests/test_*.py
+sed -i -e 's@filename="@filename="./@g' coverage.xml
+make run-pylint run-pep8

circle.yml

@@ -4,6 +4,11 @@ machine:
 
 dependencies:
   pre:
+    - pip install --upgrade pip
+    - pip install packaging
+    - pip install appdirs
+    - pip install six
+    - pip install --upgrade setuptools
     - pip install -r REQUIREMENTS.txt
     - pip install -r REQUIREMENTS_TEST.txt
     - pip install nose

debian/changelog

+python-pbcommand (1.1.1-1) unstable; urgency=medium
+
+  * Team upload.
+  * New upstream version
+  * debhelper 11
+  * Point Vcs fields to salsa.debian.org
+  * Standards-Version: 4.2.1
+  * Secure URI in copyright format
+  * Respect DEB_BUILD_OPTIONS in override_dh_auto_test target
+  * Replace needs-recommends in test restrictions by recommended packages
+
+ -- Andreas Tille <tille@debian.org>  Wed, 31 Oct 2018 19:08:15 +0100
+
 python-pbcommand (0.5.3-1) unstable; urgency=medium
 
   * Team upload.

debian/compat

-9
+11

debian/control

 Source: python-pbcommand
-Section: python
-Priority: optional
 Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
 Uploaders: Afif Elghraoui <afif@debian.org>
-Build-Depends:
-	debhelper (>= 9),
+Section: python
+Priority: optional
+Build-Depends: debhelper (>= 11~),
                dh-python,
                python-all,
                python-setuptools,
@@ -17,10 +16,10 @@ Build-Depends:
 # For tests
                python-nose,
                python-pbcore
-Standards-Version: 3.9.8
+Standards-Version: 4.2.1
+Vcs-Browser: https://salsa.debian.org/med-team/python-pbcommand
+Vcs-Git: https://salsa.debian.org/med-team/python-pbcommand.git
 Homepage: https://pbcommand.readthedocs.org/en/latest/
-Vcs-Git: https://anonscm.debian.org/git/debian-med/python-pbcommand.git
-Vcs-Browser: https://anonscm.debian.org/cgit/debian-med/python-pbcommand.git
 
 Package: python-pbcommand
 Architecture: all

debian/copyright

-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: pbcommand
 Upstream-Contact: Michael Kocher <mkocher@pacificbiosciences.com>
 Source: https://github.com/PacificBiosciences/pbcommand

debian/rules

@@ -11,5 +11,7 @@ export PYBUILD_NAME=pbcommand
 	LC_ALL=C.UTF-8 dh $@ --with python2 --buildsystem=pybuild
 
 override_dh_auto_test:
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
 	PYTHONPATH=$(CURDIR) \
 	nosetests -s --verbose --logging-config log_nose.cfg --where=tests
+endif

debian/tests/control

 Test-Command: nosetests -s --verbose --logging-config log_nose.cfg --where=tests
-Depends: @, python-nose
-Restrictions: allow-stderr, needs-recommends
+Depends: @, python-nose, python-pbcore
+Restrictions: allow-stderr

docs/source/conf.py

@@ -37,7 +37,8 @@ extensions = [
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
     'IPython.sphinxext.ipython_console_highlighting',
-    'IPython.sphinxext.ipython_directive'
+    'IPython.sphinxext.ipython_directive',
+    'nbsphinx'
 ]
 
 # For argparse

docs/source/index.rst

@@ -16,6 +16,7 @@ Contents:
    services
    report_model
    api
+   cookbook_services
 
 
 Indices and tables:

docs/source/pbcommand.services.rst

-pbcommand Client to SMRT Link Services
-######################################
+pbcommand Client to SMRT Link Services and Service Data Models
+##############################################################
 
-Python client to SMRT Link Services
+Python client to SMRT Link Services and Service Data Models
 
 
   .. automodule:: pbcommand.services
-    :members: ServiceAccessLayer
\ No newline at end of file
+    :members: ServiceAccessLayer, ServiceJob, JobTypes, JobStates, ServiceEntryPoint
+    :undoc-members:
+    :show-inheritance:

extras/make_cpp_file_types_header.py

+#!/usr/bin/env python
+
+"""
+Generate C++ preprocessor defines for PacBio file type IDs.
+"""
+
+import argparse
+import sys
+
+from pbcommand.models import FileType, FileTypes
+
+
+def run(argv):
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("out", nargs='?', default="FileTypes.h")
+    args = p.parse_args(argv[1:])
+    with open(args.out, "w") as out:
+        for ft_name in dir(FileTypes):
+            ft = getattr(FileTypes, ft_name)
+            if isinstance(ft, FileType):
+                out.write("""#define {f} "{i}"\n""".format(f=ft_name, i=ft.file_type_id))
+    print "Wrote {f}".format(f=args.out)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run(sys.argv))

pbcommand/__init__.py

-VERSION = (0, 5, 3)
+VERSION = (1, 1, 1)
 
 
 def get_version():
-    """Return the version as a string. "O.7"
+    """Return the version as a string. "1.0.0"
 
     This uses a major.minor.tiny to be compatible with semver spec.
 

pbcommand/cli/core.py

@@ -14,6 +14,7 @@ Going to do this in a new steps.
 
 
 """
+from __future__ import print_function
 import argparse
 import json
 import logging
@@ -22,6 +23,7 @@ import traceback
 import shutil
 import os
 import sys
+import errno
 
 import pbcommand
 
@@ -168,6 +170,19 @@ class TemporaryResourcesManager(object):
 
     def __enter__(self):
         for resource in self.resolved_tool_contract.task.resources:
+            if resource.type_id == ResourceTypes.TMP_DIR or resource.type_id == ResourceTypes.TMP_FILE:
+                try:
+                    dirname = os.path.dirname(os.path.realpath(resource.path))
+                    os.makedirs(dirname)
+                except EnvironmentError as e:
+                    # IOError, OSError subclass Environment and add errno
+                    # Note, dirname is not strictly defined here. If there's an
+                    # Env exception caught from the right hand this will raise NameError
+                    # and will still fail, but with a different exception type.
+                    if e.errno == errno.EEXIST and os.path.isdir(dirname):
+                        pass
+                    else:
+                        raise
             if resource.type_id == ResourceTypes.TMP_DIR:
                 os.makedirs(resource.path)
 
@@ -251,6 +266,6 @@ def pbparser_runner(argv,
     if EMIT_TOOL_CONTRACT_OPTION in argv:
         # print tool_contract
         x = json.dumps(tool_contract.to_dict(), indent=4, separators=(',', ': '))
-        print x
+        print(x)
     else:
         return pacbio_args_or_contract_runner(argv, arg_parser, args_runner_func, contract_runner_func, alog, setup_log_func)

pbcommand/cli/examples/dev_gather_fasta_app.py

@@ -39,6 +39,7 @@ def __gather_fastx(fastx_reader, fastx_writer, fastx_files, output_file):
     log.info("Completed gathering {n} files (with {x} records) to {f}".format(n=len(fastx_files), f=output_file, x=n))
     return 0
 
+
 gather_fasta = functools.partial(__gather_fastx, FastaReader, FastaWriter)
 
 
@@ -61,8 +62,6 @@ def __args_gather_runner(func, chunk_json, output_file, chunk_key):
     if not chunk_key.startswith('$chunk.'):
         chunk_key = '$chunk.' + chunk_key
         log.warn("Prepending chunk key with '$chunk.' to '{c}'".format(c=chunk_key))
-    else:
-        chunk_key = chunk_key
 
     fastx_files = _get_datum_from_chunks_by_chunk_key(chunks, chunk_key)
     _ = func(fastx_files, output_file)