.circleci/installHDF5.sh

 set -x
 set -e
 if [ ! -e prefix/lib/libhdf5.so ]; then
-  wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.12/src/hdf5-1.8.12.tar.gz
+  wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.12/src/hdf5-1.8.12.tar.gz
   tar xzf hdf5-1.8.12.tar.gz
   mkdir -p prefix
   PREFIX=$PWD/prefix

.gitignore

@@ -5,9 +5,12 @@ doc/_build
 *.pyc
 *.egg-info
 nosetests.xml
+coverage.xml
+.coverage
 
 .circleci/prefix
 .circleci/hdf5-1.8.12.tar.gz
 .circleci/hdf5-1.8.12
 
 .vagrant
+.DS_Store

LICENSES.txt → LICENSE

-PacBio "BSD clear" license:
-===========================
-
-Copyright (c) 2011-2015, Pacific Biosciences of California, Inc.
+Copyright (c) 2011-2018, Pacific Biosciences of California, Inc.
 
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
+modification, are permitted provided that the following conditions are met:
 * Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
-
 * Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.
-
 * Neither the name of Pacific Biosciences nor the names of its
   contributors may be used to endorse or promote products derived from
   this software without specific prior written permission.
 
-NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
-GRANTED BY THIS LICENSE.  THIS SOFTWARE IS PROVIDED BY PACIFIC
-BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
-WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS CONTRIBUTORS
-BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
-BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
-IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
+THIS LICENSE.  THIS SOFTWARE IS PROVIDED BY PACIFIC BIOSCIENCES AND ITS
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
+NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR
+ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 BioPython license (for pbcore/io/align/_bgzf.py):

MANIFEST.in

@@ -3,13 +3,9 @@ include *.org
 include *.txt
 include *.yml
 include Makefile
-recursive-include bin *.py
-recursive-include doc *.conf
-recursive-include doc *.css
-recursive-include doc *.jpg
-recursive-include doc *.png
-recursive-include doc *.py
-recursive-include doc *.rst
-recursive-include doc Makefile
-recursive-include pbcore Makefile
-recursive-include tests *.py
+include LICENSE
+
+graft bin
+graft doc
+graft pbcore
+graft tests

Makefile

@@ -15,7 +15,7 @@ install:
 	@pip install ./
 
 pylint:
-	pylint --errors-only pbcore/
+	pylint --errors-only --ignore=pyxb pbcore/
 
 clean: doc-clean
 	rm -rf build/;\
@@ -33,7 +33,8 @@ doctest:
 	cd doc && make doctest
 
 unit-test:
-	nosetests --with-coverage --cover-xml-file=coverage.xml --cover-package=pbcore --cover-xml --with-xunit -v tests
+	#nosetests --with-coverage --cover-xml-file=coverage.xml --cover-package=pbcore --cover-xml --with-xunit -v tests
+	pytest -v -n auto --dist=loadscope --durations=20 --junitxml=nosetests.xml --cov=./pbcore --cov-report=xml:coverage.xml tests/test_*.py
 	sed -i -e 's@filename="@filename="./@g' coverage.xml
 
 test: doctest unit-test
@@ -71,3 +72,8 @@ xsd-codegen:
 
 validate-metadata:
 	xmllint --schema ../xsd-datamodels/PacBioCollectionMetadata.xsd pbcore/data/datasets/CollectionMetadata.xml
+
+wheel:
+	which pip
+	pip wheel --wheel-dir=${WHEELHOUSE} --no-deps .
+	ls -larth ${WHEELHOUSE}

bamboo_build.sh

 #!/bin/bash
 type module >& /dev/null || . /mnt/software/Modules/current/init/bash
-module load python/2.7.9-mobs-pbcore
+module load python/2
 set -ex
-
+nproc
 
 export PATH=$PWD/build/bin:$PATH
 export PYTHONUSERBASE=$PWD/build
@@ -20,11 +20,16 @@ fi
 
 rm -rf   build
 mkdir -p build/bin build/lib build/include build/share
-$PIP install --no-compile --find-link $WHEELHOUSE --user -r requirements.txt
-$PIP install --no-compile --find-link $WHEELHOUSE --user -r requirements-dev.txt
-$PIP install --no-compile --find-link $WHEELHOUSE --user -e ./
+$PIP install --user --no-index --find-link $WHEELHOUSE --no-compile -e .[test]
+$PIP install --user --no-index --find-link $WHEELHOUSE pbtestdata
+$PIP install --user --no-index --find-link $WHEELHOUSE pytest-xdist
+$PIP install --user --no-index --find-link $WHEELHOUSE pytest-cov
+#$PIP install --user --no-index --find-link $WHEELHOUSE pytest-parallel # not sure why this fails
+pytest --trace-config --collect-only
 
 set +e
 make pylint # way too many errors right now
 set -e
 make test
+
+bash bamboo_wheel.sh

bamboo_wheel.sh

+#!/bin/bash
+type module >& /dev/null || . /mnt/software/Modules/current/init/bash
+module purge
+module load gcc
+module load ccache
+set -vex
+ls -larth ..
+ls -larth
+pwd
+
+case "${bamboo_planRepository_branchName}" in
+  develop|master)
+    echo "Building wheels ..."
+    ;;
+  *)
+    echo "Not building wheels."
+    exit
+    ;;
+esac
+
+export WHEELHOUSE=./wheelhouse
+
+# Give everybody read/write access.
+umask 0000
+
+
+module load python/2-UCS2
+make wheel
+
+# For now, we have only "any" wheels, so we do not need to build again.
+
+module unload python
+
+module load python/2-UCS4
+make wheel
+
+
+# http://bamboo.pacificbiosciences.com:8085/build/admin/edit/defaultBuildArtifact.action?buildKey=SAT-TAGDEPS-JOB1
+# For old artifact config:
+#mkdir -p ./artifacts/gcc-6.4.0/wheelhouse
+#rsync -av ${WHEELHOUSE}/falcon*.whl artifacts/gcc-6.4.0/wheelhouse/
+
+
+# Select export dir based on Bamboo branch, but only for develop and master.
+case "${bamboo_planRepository_branchName}" in
+  develop|master)
+    WHEELHOUSE="/mnt/software/p/python/wheelhouse/${bamboo_planRepository_branchName}/"
+    rsync -av ./wheelhouse/ ${WHEELHOUSE}
+    ;;
+  *)
+    ;;
+esac

debian/changelog

+python-pbcore (1.7.1+dfsg-1) UNRELEASED; urgency=medium
+
+  * Team upload.
+  * New upstream version
+  * debhelper-compat 12
+  * Drop Python2 support and switch to Python3
+    --> not supported upstream see https://github.com/PacificBiosciences/pbcore/issues/121
+  * Standards-Version: 4.4.0
+
+ -- Andreas Tille <tille@debian.org>  Wed, 04 Sep 2019 10:29:56 +0200
+
 python-pbcore (1.6.5+dfsg-1) unstable; urgency=medium
 
   [ Andreas Tille ]

debian/compat

-12

debian/control

@@ -2,43 +2,32 @@ Source: python-pbcore
 Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
 Section: python
 Priority: optional
-Build-Depends: debhelper (>= 12),
+Build-Depends: debhelper-compat (= 12),
                dh-python,
-               python-all,
-               python-setuptools,
-               python-h5py (>= 2.0.1),
-               python-numpy (>= 1.7.1),
-               python-pysam (>= 0.13),
                python3-sphinx <!nodoc>,
-               python-nose (>= 1.3.4) <!nocheck>,
-               python-pyxb (>= 1.2.4) <!nocheck>,
-               python-uritools <!nocheck>,
-#               python-parse,
-               pylint,
-               python-coverage,
-#               python3-all,
-#               python3-setuptools,
-#               python3-h5py (>= 2.0.1),
-#               python3-numpy (>= 1.7.1),
-#               python3-pysam (>= 0.13),
-#               python3-nose (>= 1.3.4) <!nocheck>,
-#               python3-pyxb (>= 1.2.4) <!nocheck>,
-#               python3-uritools <!nocheck>,
-#               pylint3,
-#               python3-coverage
-Standards-Version: 4.3.0
+               python3-all,
+               python3-setuptools,
+               python3-h5py,
+               python3-numpy,
+               python3-pysam (>= 0.13),
+               python3-nose <!nocheck>,
+               python3-pyxb <!nocheck>,
+               python3-uritools <!nocheck>,
+               pylint3,
+               python3-coverage
+Standards-Version: 4.4.0
 Vcs-Browser: https://salsa.debian.org/med-team/python-pbcore
 Vcs-Git: https://salsa.debian.org/med-team/python-pbcore.git
 Homepage: https://github.com/PacificBiosciences/pbcore
 
-Package: python-pbcore
+Package: python3-pbcore
 Architecture: all
 Depends: ${misc:Depends},
-         ${python:Depends},
-         python-pkg-resources
-Recommends: python-h5py
+         ${python3:Depends},
+         python3-pkg-resources
+Recommends: python3-h5py
 Suggests: python-pbcore-doc (= ${source:Version})
-Description: Python 2 library for processing PacBio data files
+Description: Python 3 library for processing PacBio data files
  The pbcore package provides Python modules for processing Pacific Biosciences
  data files and building PacBio bioinformatics applications. These modules
  include tools to read/write PacBio data formats, sample data files for
@@ -47,25 +36,7 @@ Description: Python 2 library for processing PacBio data files
  .
  This package is part of the SMRTAnalysis suite.
  .
- This is the Python 2 module.
-
-#Package: python3-pbcore
-#Architecture: all
-#Depends: ${misc:Depends},
-#         ${python3:Depends},
-#         python3-pkg-resources
-#Recommends: python3-h5py
-#Suggests: python-pbcore-doc (= ${source:Version})
-#Description: Python 3 library for processing PacBio data files
-# The pbcore package provides Python modules for processing Pacific Biosciences
-# data files and building PacBio bioinformatics applications. These modules
-# include tools to read/write PacBio data formats, sample data files for
-# testing and debugging, base classes, and utilities for building bioinformatics
-# applications.
-# .
-# This package is part of the SMRTAnalysis suite.
-# .
-# This is the Python 3 module.
+ This is the Python 3 module.
 
 Package: python-pbcore-doc
 Architecture: all

debian/patches/accept_debian_packaged_pylint.patch

-Author: Andreas Tille <tille@debian.org>
-Last-Update: Wed, 31 Oct 2018 15:36:38 +0100
-Description: Relax version condition and accept Debian packaged pylint
-
---- a/requirements-dev.txt
-+++ b/requirements-dev.txt
-@@ -3,4 +3,4 @@ nose==1.3.4
- pyxb >= 1.2.4
- h5py >= 2.0.1
- pysam >= 0.13
--pylint==1.6.4
-+pylint >= 1.6.4

debian/patches/enable-build-time-testing.patch

@@ -13,13 +13,3 @@ Last-Update: 2015-08-06
 +[nosetests]
 +verbosity=2
 +where=tests
---- a/setup.py
-+++ b/setup.py
-@@ -48,6 +48,7 @@ setup(
-                                'data/datasets/yieldtest/*.*']
-                                },
-     zip_safe = False,
-+    test_suite = 'nose.collector',
-     entry_points = { "console_scripts" : [ ".open = pbcore.io.opener:entryPoint" ] },
-     install_requires=_get_local_requirements(_REQUIREMENTS_FILE),
-     tests_require=_get_local_requirements(_REQUIREMENTS_TEST_FILE)

debian/patches/exclude_test_missing_data.patch

@@ -4,7 +4,7 @@ Description: Missing data for this test
 
 --- a/tests/test_pbdataset_metadata.py
 +++ b/tests/test_pbdataset_metadata.py
-@@ -165,15 +165,3 @@ class TestDataSet(unittest.TestCase):
+@@ -153,19 +153,6 @@ class TestDataSet(unittest.TestCase):
          aln = AlignmentSet(outXml)
          self.assertEqual(aln.uuid, oldUuid)
  
@@ -20,3 +20,7 @@ Description: Missing data for this test
 -        sset = SubreadSet(sset_fn)
 -        sset.metadata.collections.merge(orig_metadata.collections, forceUnique=True)
 -        self.assertEqual(len(sset.metadata.collections), 1)
+-
+     @skip_if_no_pbtestdata
+     def test_merge_biosamples(self):
+         import pbtestdata

debian/patches/fix_numpy_1.16_9c065b7.patch

-From 9c065b724ddfa2b78ddf3e2e6671cd30a50b0a66 Mon Sep 17 00:00:00 2001
-From: David Seifert <dseifert@pacificbiosciences.com>
-Date: Wed, 9 Jan 2019 17:56:40 +0100
-Subject: [PATCH] Fixes for numpy 1.16
-
-Fixes #120
----
- pbcore/io/BasH5IO.py | 5 ++++-
- 1 file changed, 4 insertions(+), 1 deletion(-)
-
-diff --git a/pbcore/io/BasH5IO.py b/pbcore/io/BasH5IO.py
-index 5db9e80..89d59e6 100644
---- a/pbcore/io/BasH5IO.py
-+++ b/pbcore/io/BasH5IO.py
-@@ -281,7 +281,10 @@ def _makeOffsetsDataStructure(h5Group):
- def _makeRegionTableIndex(regionTableHoleNumbers):
-     #  returns a dict: holeNumber -> (startRow, endRow)
-     diffs = np.ediff1d(regionTableHoleNumbers,
--                       to_begin=[1], to_end=[1])
-+                       to_begin=np.array([1],
-+                                         dtype=regionTableHoleNumbers.dtype),
-+                       to_end=np.array([1],
-+                                       dtype=regionTableHoleNumbers.dtype))
-     changepoints = np.flatnonzero(diffs)
-     startsAndEnds = zip(changepoints[:-1],
-                         changepoints[1:])

debian/patches/ignore-pyxb.patch

@@ -9,7 +9,7 @@ Last-Update: 2016-10-09
 --- a/pbcore/io/dataset/pyxb/DataSetXsd.py
 +++ b/pbcore/io/dataset/pyxb/DataSetXsd.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:a9ab3db6-517a-11e9-bbae-005056872f45')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -25,7 +25,7 @@ Last-Update: 2016-10-09
 --- a/pbcore/io/dataset/pyxb/_pbbase.py
 +++ b/pbcore/io/dataset/pyxb/_pbbase.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:a9ab3db6-517a-11e9-bbae-005056872f45')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -41,7 +41,7 @@ Last-Update: 2016-10-09
 --- a/pbcore/io/dataset/pyxb/_pbmeta.py
 +++ b/pbcore/io/dataset/pyxb/_pbmeta.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:a9ab3db6-517a-11e9-bbae-005056872f45')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -53,11 +53,11 @@ Last-Update: 2016-10-09
 +#    raise pyxb.PyXBVersionError(_PyXBVersion)
  
  # Import bindings for namespaces imported into schema
- import _pbrk as _ImportedBinding__pbrk
+ import pyxb.binding.datatypes
 --- a/pbcore/io/dataset/pyxb/_pbrk.py
 +++ b/pbcore/io/dataset/pyxb/_pbrk.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:a9ab3db6-517a-11e9-bbae-005056872f45')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -73,7 +73,7 @@ Last-Update: 2016-10-09
 --- a/pbcore/io/dataset/pyxb/_pbsample.py
 +++ b/pbcore/io/dataset/pyxb/_pbsample.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:a9ab3db6-517a-11e9-bbae-005056872f45')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -86,13 +86,3 @@ Last-Update: 2016-10-09
  
  # Import bindings for namespaces imported into schema
  import pyxb.binding.datatypes
---- a/requirements-dev.txt
-+++ b/requirements-dev.txt
-@@ -1,6 +1,6 @@
- coverage
- nose==1.3.4
--pyxb==1.2.4
-+pyxb >= 1.2.4
- sphinx
- h5py >= 2.0.1
- pysam >= 0.13

debian/patches/relax_test_constraint.patch

-Author: Andreas Tille <tille@debian.org>
-Last-Update: Mon, 25 Jun 2018 08:54:11 +0200
-Description: Relax test constraint which is rather a rounding issue
- Traceback (most recent call last):
-   File "/build/python-pbcore-1.4.2+dfsg/tests/test_pbdataset.py", line 2125, in test_stats_metadata
-     2322.16060475)
- AssertionError: 2322.1606047549185 != 2322.16060475
-
---- a/tests/test_pbdataset.py
-+++ b/tests/test_pbdataset.py
-@@ -2158,7 +2158,7 @@ class TestDataSet(unittest.TestCase):
-                          2322.805559802698)
-         self.assertEqual(ds2.metadata.summaryStats.readLenDist.sampleStd,
-                          2322.805559802698)
--        self.assertEqual(ds3.metadata.summaryStats.readLenDist.sampleStd,
-+        self.assertAlmostEqual(ds3.metadata.summaryStats.readLenDist.sampleStd,
-                          2322.16060475)
-         # uses the bins, not the previous values for mean, std, etc.:
-         self.assertEqual(ds3.metadata.summaryStats.readLenDist.sampleMed, 45)
---- a/requirements-dev.txt
-+++ b/requirements-dev.txt
-@@ -1,7 +1,6 @@
- coverage
--nose==1.3.4
-+nose >= 1.3.4
- pyxb >= 1.2.4
--sphinx
- h5py >= 2.0.1
- pysam >= 0.13
- pylint==1.6.4

debian/patches/series

 doc-theme.patch
 enable-build-time-testing.patch
 ignore-pyxb.patch
-relax_test_constraint.patch
-accept_debian_packaged_pylint.patch
 exclude_test_missing_data.patch
-# python3_support.patch
-fix_numpy_1.16_9c065b7.patch