Makefile

@@ -68,3 +68,6 @@ publish-to-pypi:
 xsd-codegen:
 	rm -f pbcore/io/dataset/pyxb/DataSetXsd.py
 	./bin/updateXSDs.py ../xsd-datamodels/PacBioDatasets.xsd pbcore/io/dataset/pyxb/
+
+validate-metadata:
+	xmllint --schema ../xsd-datamodels/PacBioCollectionMetadata.xsd pbcore/data/datasets/CollectionMetadata.xml

README.md

@@ -3,18 +3,11 @@
 The `pbcore` package provides Python APIs for interacting with PacBio
 data files and writing bioinformatics applications.
 
-## Installation:
+## Availability
+Latest version can be installed via bioconda package `pbcore`.
 
-    % pip install -r requirements.txt
-    % python setup.py install
-
-Note that h5py is now considered an "optional" dependency, since the old
-RSII sequencing files are no longer generated by current systems and
-software.  The underlying functionality has been left in place for backwards
-compatibility with existing code, but you will need to install h5py manually
-to use these parts of the API:
-
-    % pip install h5py
+Please refer to our [official pbbioconda page](https://github.com/PacificBiosciences/pbbioconda)
+for information on Installation, Support, License, Copyright, and Disclaimer.
 
 ## Documentation:
 

bamboo_build.sh

 #!/bin/bash
 type module >& /dev/null || . /mnt/software/Modules/current/init/bash
+module load python/2.7.9-mobs-pbcore
 set -ex
 
-NX3PBASEURL=http://nexus/repository/unsupported/pitchfork/gcc-6.4.0
-export PATH=$PWD/build/bin:/mnt/software/a/anaconda2/4.2.0/bin:$PATH
+
+export PATH=$PWD/build/bin:$PATH
 export PYTHONUSERBASE=$PWD/build
-export CFLAGS="-I/mnt/software/a/anaconda2/4.2.0/include"
-PIP="pip --cache-dir=$bamboo_build_working_directory/.pip"
-module load gcc
+
+PIP="pip --cache-dir=${bamboo_build_working_directory:-$PWD}/.pip"
+if [[ -z ${bamboo_repository_branch_name+x} ]]; then
+  WHEELHOUSE=/mnt/software/p/python/wheelhouse/develop
+elif [[ ${bamboo_repository_branch_name} == develop ]]; then
+  WHEELHOUSE=/mnt/software/p/python/wheelhouse/develop
+elif [[ ${bamboo_repository_branch_name} == master ]]; then
+  WHEELHOUSE=/mnt/software/p/python/wheelhouse/master
+else
+  WHEELHOUSE=/mnt/software/p/python/wheelhouse/develop
+fi
 
 rm -rf   build
 mkdir -p build/bin build/lib build/include build/share
-$PIP install --user \
-  $NX3PBASEURL/pythonpkgs/pysam-0.13-cp27-cp27mu-linux_x86_64.whl
-$PIP install --user -r requirements.txt
-$PIP install --user -r requirements-dev.txt
-$PIP install --user -e ./    
+$PIP install --no-compile --find-link $WHEELHOUSE --user -r requirements.txt
+$PIP install --no-compile --find-link $WHEELHOUSE --user -r requirements-dev.txt
+$PIP install --no-compile --find-link $WHEELHOUSE --user -e ./
 
 set +e
 make pylint # way too many errors right now

debian/changelog

+python-pbcore (1.6.5+dfsg-1) UNRELEASED; urgency=medium
+
+  * Team upload.
+  * Upstream is now tagging releases - adjust watch file
+  * Standards-Version: 4.2.1
+  * Secure URI in copyright format
+  * Respect DEB_BUILD_OPTIONS in override_dh_auto_test target
+  * Remove trailing whitespace in debian/changelog
+  * Build-Depends: s/python-sphinx/python3-sphinx/
+
+ -- Andreas Tille <tille@debian.org>  Wed, 31 Oct 2018 15:30:29 +0100
+
 python-pbcore (1.5.0+git20180606.43fcd9d+dfsg-3) UNRELEASED; urgency=medium
 
   * Team upload.

debian/control

@@ -10,11 +10,11 @@ Build-Depends: debhelper (>= 11~),
                python-h5py,
                python-numpy,
                python-pysam (>= 0.9.0),
-               python-sphinx <!nodoc>,
+               python3-sphinx <!nodoc>,
                python-nose <!nocheck>,
                python-pyxb <!nocheck>,
                python-coverage
-Standards-Version: 4.1.4
+Standards-Version: 4.2.1
 Vcs-Browser: https://salsa.debian.org/med-team/python-pbcore
 Vcs-Git: https://salsa.debian.org/med-team/python-pbcore.git
 Homepage: https://github.com/PacificBiosciences/pbcore

debian/copyright

-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: pbcore
 Source: https://github.com/PacificBiosciences/pbcore
 Files-Excluded: doc/pacbio-theme/*

debian/patches/ignore-pyxb.patch

@@ -9,7 +9,7 @@ Last-Update: 2016-10-09
 --- a/pbcore/io/dataset/pyxb/DataSetXsd.py
 +++ b/pbcore/io/dataset/pyxb/DataSetXsd.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:4c2de410-68d0-11e8-b8c4-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -21,11 +21,11 @@ Last-Update: 2016-10-09
 +#    raise pyxb.PyXBVersionError(_PyXBVersion)
  
  # Import bindings for namespaces imported into schema
- import _pbbase as _ImportedBinding__pbbase
+ import pyxb.binding.datatypes
 --- a/pbcore/io/dataset/pyxb/_pbbase.py
 +++ b/pbcore/io/dataset/pyxb/_pbbase.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:4c2de410-68d0-11e8-b8c4-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -41,7 +41,7 @@ Last-Update: 2016-10-09
 --- a/pbcore/io/dataset/pyxb/_pbmeta.py
 +++ b/pbcore/io/dataset/pyxb/_pbmeta.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:4c2de410-68d0-11e8-b8c4-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -53,11 +53,11 @@ Last-Update: 2016-10-09
 +#    raise pyxb.PyXBVersionError(_PyXBVersion)
  
  # Import bindings for namespaces imported into schema
- import _pbbase as _ImportedBinding__pbbase
+ import _pbrk as _ImportedBinding__pbrk
 --- a/pbcore/io/dataset/pyxb/_pbrk.py
 +++ b/pbcore/io/dataset/pyxb/_pbrk.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:4c2de410-68d0-11e8-b8c4-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -69,11 +69,11 @@ Last-Update: 2016-10-09
 +#    raise pyxb.PyXBVersionError(_PyXBVersion)
  
  # Import bindings for namespaces imported into schema
- import _pbbase as _ImportedBinding__pbbase
+ import pyxb.binding.datatypes
 --- a/pbcore/io/dataset/pyxb/_pbsample.py
 +++ b/pbcore/io/dataset/pyxb/_pbsample.py
 @@ -18,10 +18,10 @@ import pyxb.utils.six as _six
- _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:4c2de410-68d0-11e8-b8c4-1803730e031b')
+ _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:e2ef7b8c-c340-11e8-a616-1803730e031b')
  
  # Version of PyXB used to generate the bindings
 -_PyXBVersion = '1.2.4'
@@ -85,14 +85,4 @@ Last-Update: 2016-10-09
 +#    raise pyxb.PyXBVersionError(_PyXBVersion)
  
  # Import bindings for namespaces imported into schema
- import _pbbase as _ImportedBinding__pbbase
---- a/requirements-dev.txt
-+++ b/requirements-dev.txt
-@@ -1,6 +1,6 @@
- coverage
- nose
--pyxb==1.2.4
-+pyxb
- sphinx
- h5py >= 2.0.1
- pysam >= 0.13
+ import pyxb.binding.datatypes

debian/patches/relax_test_constraint.patch

@@ -8,7 +8,7 @@ Description: Relax test constraint which is rather a rounding issue
 
 --- a/tests/test_pbdataset.py
 +++ b/tests/test_pbdataset.py
-@@ -2122,7 +2122,7 @@ class TestDataSet(unittest.TestCase):
+@@ -2158,7 +2158,7 @@ class TestDataSet(unittest.TestCase):
                           2322.805559802698)
          self.assertEqual(ds2.metadata.summaryStats.readLenDist.sampleStd,
                           2322.805559802698)

debian/patches/remove-unnecessary-cython-dep.patch

-Description: Remove cython from requirements.txt
- Upstream now reads build dependencies from this file,
- but Cython isn't needed and setup.py therefore tries
- to download it (unsuccessfully).
-Author: Afif Elghraoui <afif@ghraoui.name
-Forwarded: no
-Last-Update: 2015-12-13
---- a/requirements.txt
-+++ b/requirements.txt
-@@ -1,3 +1,2 @@
--cython
- numpy >= 1.7.1
- pysam >= 0.13

debian/patches/series

 doc-theme.patch
 enable-build-time-testing.patch
-remove-unnecessary-cython-dep.patch
 ignore-pyxb.patch
 relax_test_constraint.patch

debian/rules

@@ -29,9 +29,11 @@ override_dh_auto_clean:
 	$(MAKE) clean
 
 override_dh_auto_test:
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
 	PYBUILD_SYSTEM=custom \
 	PYBUILD_TEST_ARGS="{interpreter} setup.py test" \
 	  dh_auto_test
+endif
 
 override_dh_installchangelogs:
 	dh_installchangelogs CHANGELOG.org

debian/watch

 version=4
 
+opts="repacksuffix=+dfsg,dversionmangle=s/\+dfsg//g,repack,compression=xz" \
+   https://github.com/PacificBiosciences/pbcore/releases .*/archive/v?@ANY_VERSION@@ARCHIVE_EXT@
+
 # Upstream is not tagging releases regularly - thus we use mode=git
-opts="mode=git,pretty=1.5.0+git%cd.%h,repacksuffix=+dfsg,dversionmangle=s/\+dfsg//g,repack,compression=xz" \
-  https://github.com/PacificBiosciences/pbcore.git HEAD
+#opts="mode=git,pretty=1.5.5+git%cd.%h,repacksuffix=+dfsg,dversionmangle=s/\+dfsg//g,repack,compression=xz" \
+#  https://github.com/PacificBiosciences/pbcore.git HEAD

doc/conf.py

@@ -14,7 +14,7 @@
 import sys, os
 
 # don't forget to update setup.py and pbcore/__init__.py too
-__VERSION__ = '1.5.0'
+__VERSION__ = '1.6.5'
 
 
 # If extensions (or modules to document with autodoc) are in another directory,

pbcore/__init__.py

 from __future__ import absolute_import
 
 # don't forget to update setup.py and doc/conf.py too
-__VERSION__ = "1.5.0"
+__VERSION__ = "1.6.5"

pbcore/chemistry/resources/mapping.xml

@@ -377,19 +377,19 @@
     <SoftwareVersion>5.0</SoftwareVersion>
   </Mapping>
 
-  <!-- SG1/16509P Chemistry update (binding kit only) with 5.0 basecaller (no new training, prototype parts) -->
+  <!-- 5.0.1 ChemRel; Sequel® Binding Kit 3.0; Sequel® Sequencing Plate 3.0 (8 rxn) -->
   <Mapping>
-    <SequencingChemistry>S/P2-C2/5.0</SequencingChemistry>
+    <SequencingChemistry>S/P3-C3/5.0</SequencingChemistry>
     <BindingKit>101-500-400</BindingKit>
-    <SequencingKit>101-490-900</SequencingKit>
+    <SequencingKit>101-427-500</SequencingKit>
     <SoftwareVersion>5.0</SoftwareVersion>
   </Mapping>
 
-  <!-- SG1/16509P Chemistry update (binding kit only) with 5.0 basecaller (no new training, prototype parts) -->
+  <!-- 5.0.1 ChemRel; Sequel® Binding Kit 3.0; Sequel® Sequencing Plate 3.0 (4 rxn)  -->
   <Mapping>
-    <SequencingChemistry>S/P2-C2/5.0</SequencingChemistry>
+    <SequencingChemistry>S/P3-C3/5.0</SequencingChemistry>
     <BindingKit>101-500-400</BindingKit>
-    <SequencingKit>101-491-000</SequencingKit>
+    <SequencingKit>101-427-800</SequencingKit>
     <SoftwareVersion>5.0</SoftwareVersion>
   </Mapping>
 

pbcore/data/datasets/CollectionMetadata.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<pbmeta:Collections xmlns="http://pacificbiosciences.com/PacBioDatasets.xsd" xmlns:pbbase="http://pacificbiosciences.com/PacBioBaseDataModel.xsd" xmlns:pbmeta="http://pacificbiosciences.com/PacBioCollectionMetadata.xsd" xmlns:pbsample="http://pacificbiosciences.com/PacBioSampleInfo.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pacificbiosciences.com/PacBioDataModel.xsd">
+  <pbmeta:CollectionMetadata Context="unknown" InstrumentId="1" InstrumentName="unknown" MetaType="PacBio.Collection" TimeStampedName="unknown" UniqueId="2d72e3db-49b5-4d72-aa3e-0dfa05bf3734">
+    <pbmeta:InstCtrlVer>0.0.0</pbmeta:InstCtrlVer>
+    <pbmeta:SigProcVer>unknown</pbmeta:SigProcVer>
+    <pbmeta:RunDetails>
+      <pbmeta:TimeStampedName>unknown</pbmeta:TimeStampedName>
+      <pbmeta:Name>unknown</pbmeta:Name>
+    </pbmeta:RunDetails>
+    <pbmeta:WellSample Name="unknown">
+      <pbmeta:WellName>unknown</pbmeta:WellName>
+      <pbmeta:Concentration>0.0</pbmeta:Concentration>
+      <pbmeta:InsertSize>2000</pbmeta:InsertSize>
+      <pbmeta:SampleReuseEnabled>false</pbmeta:SampleReuseEnabled>
+      <pbmeta:StageHotstartEnabled>true</pbmeta:StageHotstartEnabled>
+      <pbmeta:SizeSelectionEnabled>false</pbmeta:SizeSelectionEnabled>
+      <pbmeta:UseCount>1</pbmeta:UseCount>
+      <pbsample:BioSamples>
+        <pbsample:BioSample Name="unknown">
+        </pbsample:BioSample>
+      </pbsample:BioSamples>
+    </pbmeta:WellSample>
+    <pbmeta:Automation>
+      <pbbase:AutomationParameters>
+        <pbbase:AutomationParameter/>
+      </pbbase:AutomationParameters>
+    </pbmeta:Automation>
+    <pbmeta:CollectionNumber>1</pbmeta:CollectionNumber>
+    <pbmeta:CellIndex>1</pbmeta:CellIndex>
+    <pbmeta:Primary>
+      <pbmeta:AutomationName>unknown</pbmeta:AutomationName>
+      <pbmeta:ConfigFileName>unknown</pbmeta:ConfigFileName>
+      <pbmeta:SequencingCondition/>
+      <pbmeta:OutputOptions>
+        <pbmeta:ResultsFolder>Analysis_Results</pbmeta:ResultsFolder>
+        <pbmeta:CollectionPathUri>unknown</pbmeta:CollectionPathUri>
+        <pbmeta:CopyFiles>
+          <pbmeta:CollectionFileCopy>Fasta</pbmeta:CollectionFileCopy>
+        </pbmeta:CopyFiles>
+        <pbmeta:Readout>Bases</pbmeta:Readout>
+        <pbmeta:MetricsVerbosity>Minimal</pbmeta:MetricsVerbosity>
+      </pbmeta:OutputOptions>
+    </pbmeta:Primary>
+  </pbmeta:CollectionMetadata>
+</pbmeta:Collections>

pbcore/data/datasets/__init__.py

@@ -25,6 +25,7 @@ XML_FILES = ["alignment.dataset.xml", #0
              "pbalchemysim1.hdfsubreadset.xml",
              "pbalchemysim.hdfsubreadset.xml", #19
              "ccsaligned.dataset.xml",
+             "CollectionMetadata.xml"
             ]
 BAM_FILES = ["pbalchemysim0.pbalign.bam",
              "pbalchemysim1.pbalign.bam",
@@ -68,3 +69,6 @@ def getHdfSubreadSet():
 
 def getBarcodedBam():
     return _getAbsPath(BAM_FILES[3])
+
+def getMockCollectionMetadata():
+    return _getAbsPath(XML_FILES[21])

pbcore/io/BasH5IO.py

@@ -596,8 +596,9 @@ class BasH5Reader(object):
          are the subintervals of the raw read corresponding to the
          SMRTbell insert::
 
+           >>> from __future__ import print_function
            >>> subreads = zmw8.subreads
-           >>> print subreads
+           >>> print(subreads)
            [<ZmwRead: m110818_075520_42141_c100129202555500000315043109121112_s1_p0/8/3381_3881>,
            <ZmwRead: m110818_075520_42141_c100129202555500000315043109121112_s1_p0/8/3924_4398>,
            <ZmwRead: m110818_075520_42141_c100129202555500000315043109121112_s1_p0/8/4445_4873>,

pbcore/io/FastaIO.py

@@ -167,12 +167,13 @@ class FastaReader(ReaderBase):
 
     .. doctest::
 
+        >>> from __future__ import print_function
         >>> from pbcore.io import FastaReader
         >>> from pbcore import data
         >>> filename = data.getTinyFasta()
         >>> r = FastaReader(filename)
         >>> for record in r:
-        ...     print record.header, len(record.sequence)
+        ...     print(record.header, len(record.sequence))
         ref000001|EGFR_Exon_2 183
         ref000002|EGFR_Exon_3 203
         ref000003|EGFR_Exon_4 215
@@ -386,11 +387,12 @@ class IndexedFastaReader(ReaderBase, Sequence):
 
     .. doctest::
 
+        >>> from __future__ import print_function
         >>> from pbcore.io import FastaTable
         >>> from pbcore import data
         >>> filename = data.getFasta()
         >>> t = IndexedFastaReader(filename)
-        >>> print t[:4] # doctest: +NORMALIZE_WHITESPACE
+        >>> print(t[:4]) # doctest: +NORMALIZE_WHITESPACE
         [<IndexedFastaRecord: ref000001|EGFR_Exon_2>,
          <IndexedFastaRecord: ref000002|EGFR_Exon_3>,
          <IndexedFastaRecord: ref000003|EGFR_Exon_4>,

pbcore/io/align/BamAlignment.py

@@ -523,7 +523,10 @@ class BamAlignment(AlignmentRecordMixin):
             raise ValueError("Bad `orientation` value")
         if self.isUnmapped and (orientation != "native" or aligned == True):
             raise UnavailableFeature("Cannot get genome oriented/aligned features from unmapped BAM record")
-        data = np.fromstring(self.peer.seq, dtype=np.int8)
+        seq = self.peer.seq
+        if seq is None:
+            seq = ''
+        data = np.fromstring(seq, dtype=np.int8)
         if self.isCCS or self.isTranscript:
             s = self.aStart
             e = self.aEnd