.gitignore

@@ -17,11 +17,13 @@ tests/cbcf_data
 tests/tabix_data
 
 samtools/config.h
-bcftools/config.g
+bcftools/config.h
 htslib/config.status
 htslib/config.h
 htslib/config.log
 htslib/config.mk
+htslib/htslib.pc.tmp
+htslib/htslib-uninstalled.pc
 pysam/config.py
 
 # linking tests

.travis.yml

 os:
-  - linx
+  - linux
   - osx
 
 language: c
@@ -7,11 +7,58 @@ language: c
 env:
   matrix:
     - CONDA_PY=2.7
-    - CONDA_PY=3.4
-    - CONDA_PY=3.5
     - CONDA_PY=3.6
+    - CONDA_PY=3.7
   global:
     - PYSAM_LINKING_TEST=1
+    - TWINE_USERNAME=grepall
+    - secure: 'OcwwP8/o21+SGW0UVAnnCQwllhGSCq2HJzpI9EhX3kh6J9RTkyx/+drkg45bx1Z5u8zymuAFappEYzlpzqZE886XezkjOYGVa/u+Coqr1oT/BEJHFCkCA4o26yESp7Zy8aNj/juhB7Rfa77pIDXBayqTzbALz/AURMtZapasB18='
+
+_deploy_common: &deploy_common
+  if: tag IS present
+  install:
+    - python3 -m pip install cibuildwheel twine
+
+matrix:
+  include:
+    - stage: deploy
+      os: linux
+      language: python
+      python: '3.5'
+      services:
+        - docker
+      env:
+        - CIBW_BEFORE_BUILD="yum install -y zlib-devel bzip2-devel xz-devel && pip install -r requirements.txt"
+        - CIBW_ENVIRONMENT='HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"'
+      addons:
+        apt:
+          packages:
+            - gcc
+            - g++
+            - libcurl4-openssl-dev  # for libcurl support in sdist
+            - libssl-dev  # for s3 support in sdist
+      <<: *deploy_common
+      script:
+        - set -e
+        - cibuildwheel --output-dir dist
+        - python3 -m pip install Cython
+        - python3 setup.py build_ext --inplace
+        - python3 setup.py sdist
+        - twine check dist/*
+        - twine upload --skip-existing dist/*
+    - stage: deploy
+      os: osx
+      language: generic
+      env:
+        - CIBW_BEFORE_BUILD="pip install -r requirements.txt"
+        - CIBW_ENVIRONMENT='HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"'
+      addons: {}
+      <<: *deploy_common
+      script:
+        - set -e
+        - cibuildwheel --output-dir dist
+        - twine check dist/*
+        - twine upload --skip-existing dist/*
 
 addons:
   apt:

NEWS

@@ -5,6 +5,26 @@ http://pysam.readthedocs.io/en/latest/release.html
 Release notes
 =============
 
+Release 0.15.3
+==============
+
+Bugfix release.
+
+* [#824] allow reading of UTF-8 encoded text in VCF/BCF files.
+* [#780] close all filehandles before opening new ones in pysam_dispatch
+* [#773] do not cache VariantRecord.id to avoid memory leak
+* [#781] default of multiple_iterators=True is changed to False for
+  CRAM files.
+* [#825] fix collections.abc import
+* [#825] use bcf_hdr_format instead of bcf_hdr_fmt_text, fix memcpy
+  bug when setting FORMAT fields.
+* [#804] Use HTSlib's kstring_t, which reallocates and enlarges its
+  memory as needed, rather than a fixed-size char buffer.
+* [#814] Build wheels and upload them to PyPI
+* [#755] Allow passing flags and arguments to index methods
+* [#763] Strip \0 in header check
+* [#761] Test Tabix index contents, not the compression
+
 Release 0.15.2
 ==============
 

README.rst

@@ -25,6 +25,8 @@ as it resolves non-python dependencies and uses pre-configured
 compilation options. Especially for OS X this will potentially save a
 lot of trouble.
 
+The current version of pysam wraps 3rd-party code from htslib-1.9, samtools-1.9, and bcftools-1.9.
+
 Pysam is available through `pypi
 <https://pypi.python.org/pypi/pysam>`_. To install, type::
 

bcftools/bcftools.pysam.c

@@ -10,7 +10,6 @@
 FILE * bcftools_stderr = NULL;
 FILE * bcftools_stdout = NULL;
 const char * bcftools_stdout_fn = NULL;
-int bcftools_stdout_fileno = STDOUT_FILENO;
 
 
 FILE * bcftools_set_stderr(int fd)
@@ -21,11 +20,10 @@ FILE * bcftools_set_stderr(int fd)
   return bcftools_stderr;
 }
 
-void bcftools_unset_stderr(void)
+void bcftools_close_stderr(void)
 {
-  if (bcftools_stderr != NULL)
   fclose(bcftools_stderr);
-  bcftools_stderr = fopen("/dev/null", "w");
+  bcftools_stderr = NULL;
 }
 
 FILE * bcftools_set_stdout(int fd)
@@ -37,7 +35,6 @@ FILE * bcftools_set_stdout(int fd)
     {
       fprintf(bcftools_stderr, "could not set stdout to fd %i", fd);
     }
-  bcftools_stdout_fileno = fd;
   return bcftools_stdout;
 }
 
@@ -46,12 +43,10 @@ void bcftools_set_stdout_fn(const char *fn)
   bcftools_stdout_fn = fn;
 }
 
-void bcftools_unset_stdout(void)
+void bcftools_close_stdout(void)
 {
-  if (bcftools_stdout != NULL)
   fclose(bcftools_stdout);
-  bcftools_stdout = fopen("/dev/null", "w");
-  bcftools_stdout_fileno = STDOUT_FILENO;
+  bcftools_stdout = NULL;
 }
 
 int bcftools_puts(const char *s)

bcftools/bcftools.pysam.h

-#ifndef PYSAM_H
-#define PYSAM_H
+#ifndef bcftools_PYSAM_H
+#define bcftools_PYSAM_H
 
-#include "stdio.h"
+#include <stdio.h>
 
 extern FILE * bcftools_stderr;
 
@@ -17,7 +17,7 @@ FILE * bcftools_set_stderr(int fd);
 
 /*! set pysam standard output to point to file descriptor
 
-  Setting the stderr will close the previous stdout.
+  Setting the stdout will close the previous stdout.
  */
 FILE * bcftools_set_stdout(int fd);
 
@@ -26,17 +26,15 @@ FILE * bcftools_set_stdout(int fd);
  */
 void bcftools_set_stdout_fn(const char * fn);
 
-/*! set pysam standard error to /dev/null.
+/*! close pysam standard error and set to NULL
   
-  Unsetting the stderr will close the previous stderr.
  */
-void bcftools_unset_stderr(void);
+void bcftools_close_stderr(void);
 
-/*! set pysam standard error to /dev/null.
+/*! close pysam standard output and set to NULL
   
-  Unsetting the stderr will close the previous stderr.
  */
-void bcftools_unset_stdout(void);
+void bcftools_close_stdout(void);
 
 int bcftools_puts(const char *s);
 

debian/changelog

+python-pysam (0.15.3+ds-1) UNRELEASED; urgency=medium
+
+  * Team upload.
+  * New upstream release. Not yet dropping py2 due to paleomix, sga, others
+  * debhelper-compat 12
+  * Standards-Version: 4.4.0
+  * Trim trailing whitespace.
+  * Use secure URI in Homepage field.
+  * Set upstream metadata fields: Repository.
+
+ -- Michael R. Crusoe <michael.crusoe@gmail.com>  Tue, 06 Aug 2019 19:37:44 +0200
+
 python-pysam (0.15.2+ds-2) unstable; urgency=medium
 
   * Team upload.

debian/compat

-11

debian/control

 Source: python-pysam
 Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
 Uploaders: Charles Plessy <plessy@debian.org>,
-           Andreas Tille <tille@debian.org>,
+           Andreas Tille <tille@debian.org>
 Section: python
 Testsuite: autopkgtest-pkg-python
 Priority: optional
-Build-Depends: debhelper (>= 11~),
+Build-Depends: debhelper-compat (= 12),
                dh-exec,
                dh-python,
                libhts-dev (>= 1.9),
@@ -21,10 +21,10 @@ Build-Depends: debhelper (>= 11~),
                bcftools (>= 1.9) <!nocheck>,
                python-pytest <!nocheck>,
                python3-pytest <!nocheck>
-Standards-Version: 4.3.0
+Standards-Version: 4.4.0
 Vcs-Browser: https://salsa.debian.org/med-team/python-pysam
 Vcs-Git: https://salsa.debian.org/med-team/python-pysam.git
-Homepage: http://pysam.readthedocs.org/en/latest
+Homepage: https://pysam.readthedocs.org/en/latest
 
 Package: python-pysam
 Architecture: any

debian/patches/series

 skip_test_remote.patch
-skip_test_needing_missing_data.patch
 spelling
-test_index_not_compression

debian/patches/skip_test_needing_missing_data.patch

-Description: do not depend from non-existing data file in make test target
- (There is no such file example_reverse_complement.bam)
-Author: Andreas Tille <tille@debian.org>
-Last-Update: Sat, 17 Feb 2018 15:24:34 +0100
-Forwarded: https://github.com/pysam-developers/pysam/issues/626
-
-Index: python-pysam/tests/AlignmentFile_test.py
-===================================================================
---- python-pysam.orig/tests/AlignmentFile_test.py
-+++ python-pysam/tests/AlignmentFile_test.py
-@@ -1402,11 +1402,6 @@ class TestEmptyHeader(unittest.TestCase)
-                                              'example_empty_header.bam'))
-         self.assertEqual(s.header.to_dict(), {'SQ': [{'LN': 1000, 'SN': 'chr1'}]})
- 
--    def test_bam_without_seq_in_header(self):
--        s = pysam.AlignmentFile(os.path.join(BAM_DATADIR, "example_no_seq_in_header.bam"))
--        self.assertTrue("SQ" in s.header.to_dict())
--        self.assertTrue("@SQ" in str(s.header))
--
- 
- class TestMismatchingHeader(unittest.TestCase):
-     '''see issue 716.'''

debian/patches/skip_test_remote.patch

@@ -2,11 +2,9 @@ Author: Andreas Tille <tille@debian.org>
 Last-Update: Tue, 11 Sep 2018 14:12:55 +0200
 Description: Skip tests trying to access remote site
 
-Index: python-pysam/tests/AlignmentFile_test.py
-===================================================================
 --- python-pysam.orig/tests/AlignmentFile_test.py
 +++ python-pysam/tests/AlignmentFile_test.py
-@@ -1623,6 +1623,7 @@ class TestDoubleFetchCRAMWithReference(T
+@@ -1631,6 +1631,7 @@
      reference_filename = os.path.join(BAM_DATADIR, 'ex1.fa')
  
  
@@ -14,11 +12,9 @@ Index: python-pysam/tests/AlignmentFile_test.py
  class TestRemoteFileFTP(unittest.TestCase):
  
      '''test remote access.
-Index: python-pysam/tests/tabix_test.py
-===================================================================
 --- python-pysam.orig/tests/tabix_test.py
 +++ python-pysam/tests/tabix_test.py
-@@ -1039,6 +1039,7 @@ for vcf_file in vcf_files:
+@@ -1014,6 +1014,7 @@
      globals()[n] = type(n, (TestVCFFromVariantFile,), dict(filename=vcf_file,))
  
  
@@ -26,7 +22,7 @@ Index: python-pysam/tests/tabix_test.py
  class TestRemoteFileHTTP(unittest.TestCase):
  
      url = "http://genserv.anat.ox.ac.uk/downloads/pysam/test/example.gtf.gz"
-@@ -1078,25 +1079,28 @@ class TestRemoteFileHTTP(unittest.TestCa
+@@ -1053,25 +1054,28 @@
          self.assertEqual(list(self.local_file.header), [])
  
  

debian/patches/spelling

@@ -24,7 +24,7 @@ Subject: Fix spelling typos, courtesy of lintian
  
 --- python-pysam.orig/pysam/libcalignedsegment.pyx
 +++ python-pysam/pysam/libcalignedsegment.pyx
-@@ -2244,7 +2244,7 @@
+@@ -2238,7 +2238,7 @@
          *value*.
  
          An existing value of the same *tag* will be overwritten unless
@@ -35,7 +35,7 @@ Subject: Fix spelling typos, courtesy of lintian
          If *value* is None, the tag will be deleted.
 --- python-pysam.orig/pysam/libcalignmentfile.pyx
 +++ python-pysam/pysam/libcalignmentfile.pyx
-@@ -1023,7 +1023,7 @@
+@@ -1028,7 +1028,7 @@
  
          See :meth:`~pysam.HTSFile.parse_region` for more information
          on how genomic regions can be specified. :term:`reference` and
@@ -44,7 +44,7 @@ Subject: Fix spelling typos, courtesy of lintian
          for :term:`contig` and `stop`, respectively.
  
          Without a `contig` or `region` all mapped reads in the file
-@@ -1206,7 +1206,7 @@
+@@ -1211,7 +1211,7 @@
          """perform a :term:`pileup` within a :term:`region`. The region is
          specified by :term:`contig`, `start` and `stop` (using
          0-based indexing).  :term:`reference` and `end` are also accepted for
@@ -53,7 +53,7 @@ Subject: Fix spelling typos, courtesy of lintian
          respectively.  Alternatively, a samtools 'region' string
          can be supplied.
  
-@@ -1355,7 +1355,7 @@
+@@ -1360,7 +1360,7 @@
  
          The region is specified by :term:`contig`, `start` and `stop`.
          :term:`reference` and `end` are also accepted for backward
@@ -62,7 +62,7 @@ Subject: Fix spelling typos, courtesy of lintian
          respectively.  Alternatively, a :term:`samtools` :term:`region`
          string can be supplied.
  
-@@ -1459,7 +1459,7 @@
+@@ -1464,7 +1464,7 @@
  
          The region is specified by :term:`contig`, `start` and `stop`.
          :term:`reference` and `end` are also accepted for backward
@@ -73,7 +73,7 @@ Subject: Fix spelling typos, courtesy of lintian
  
 --- python-pysam.orig/pysam/libchtslib.pxd
 +++ python-pysam/pysam/libchtslib.pxd
-@@ -2502,7 +2502,7 @@
+@@ -2513,7 +2513,7 @@
      #         2 if the file is a stream and thus unseekable
      #         1 if the file contains an EOF block
      #         0 if the file does not contain an EOF block

debian/patches/test_index_not_compression

-From: Michael R. Crusoe <michael.crusoe@gmail.com>
-Subject: Test Tabix index contents, not the compression
-
-Fixes: https://github.com/samtools/htslib/issues/827
-
---- python-pysam.orig/tests/tabix_test.py
-+++ python-pysam/tests/tabix_test.py
-@@ -14,8 +14,6 @@
- import subprocess
- import glob
- import re
--import copy
--import tempfile
- from TestUtils import check_url, load_and_convert, TABIX_DATADIR, get_temp_filename
- 
- IS_PYTHON3 = sys.version_info[0] >= 3
-@@ -64,6 +62,17 @@
-     return found
- 
- 
-+def checkGZBinaryEqual(filename1, filename2):
-+    '''return true if the two files are binary equal.'''
-+    with gzip.open(filename1, "rb") as infile1:
-+        d1 = infile1.read()
-+        with gzip.open(filename2, "rb") as infile2:
-+            d2 = infile2.read()
-+        if d1 == d2:
-+            return True
-+    return False
-+
-+
- class TestIndexing(unittest.TestCase):
-     filename = os.path.join(TABIX_DATADIR, "example.gtf.gz")
-     filename_idx = os.path.join(TABIX_DATADIR, "example.gtf.gz.tbi")
-@@ -77,7 +86,7 @@
-         '''test indexing via preset.'''
- 
-         pysam.tabix_index(self.tmpfilename, preset="gff")
--        self.assertTrue(checkBinaryEqual(
-+        self.assertTrue(checkGZBinaryEqual(
-             self.tmpfilename + ".tbi", self.filename_idx))
- 
-     def test_indexing_to_custom_location_works(self):
-@@ -86,7 +95,7 @@
-         index_path = get_temp_filename(suffix='custom.tbi')
-         pysam.tabix_index(self.tmpfilename, preset="gff",
-                           index=index_path, force=True)
--        self.assertTrue(checkBinaryEqual(index_path, self.filename_idx))
-+        self.assertTrue(checkGZBinaryEqual(index_path, self.filename_idx))
-         os.unlink(index_path)
- 
-     def test_indexing_with_explict_columns_works(self):
-@@ -98,7 +107,7 @@
-                           end_col=4,
-                           line_skip=0,
-                           zerobased=False)
--        self.assertTrue(checkBinaryEqual(
-+        self.assertTrue(checkGZBinaryEqual(
-             self.tmpfilename + ".tbi", self.filename_idx))
- 
-     def test_indexing_with_lineskipping_works(self):
-@@ -109,7 +118,7 @@
-                           end_col=4,
-                           line_skip=1,
-                           zerobased=False)
--        self.assertFalse(checkBinaryEqual(
-+        self.assertFalse(checkGZBinaryEqual(
-             self.tmpfilename + ".tbi", self.filename_idx))
- 
-     def tearDown(self):

debian/rules

@@ -72,4 +72,3 @@ clean-tests: pysam_data.clean cbcf_data.clean
 		tests/pysam_data/ex1.fa.gz.gzi \
 		tests/pysam_data/ex1_csi.bam.csi
 	rm -rf .pytest_cache/
-

debian/tests/control

-Test-Command: make -C tests/pysam_data && make -C tests/cbcf_data && pytest
+Test-Command: export HOME=$PWD && make -C tests/pysam_data && make -C tests/cbcf_data && pytest
 Depends: @builddeps@, python-pysam
 Restrictions: allow-stderr, rw-build-tree
 
-Test-Command: make -C tests/pysam_data && make -C tests/cbcf_data && pytest-3
+Test-Command: export HOME=$PWD && make -C tests/pysam_data && make -C tests/cbcf_data && pytest-3
 Depends: @builddeps@, python3-pysam,
 Restrictions: allow-stderr, rw-build-tree

debian/tests/run-nose-tests

-#!/bin/sh -e
-
-if [ "$ADTTMP" = "" ] ; then
-  ADTTMP=`mktemp -d /tmp/python-pysam-test.XXXXXX`
-fi
-cp -ra /usr/share/doc/python-pysam/data/* $ADTTMP
-
-# FIXME!!
-# That's a pretty strange hack but without it the dynamic libraries are not found
-# Need to be tracked down before uploading
-cd /usr/lib/python2.7/dist-packages/pysam
-gnutype=`dpkg-architecture -qDEB_TARGET_GNU_TYPE`
-for so in *.${gnutype}.so ; do sudo ln -sf $so `basename $so .${gnutype}.so`.so ; done
-
-cd $ADTTMP
-find . -name "*.gz" -exec gunzip -f \{\} \;
-
-nosetests --nocapture -v
-cd
-# rm -rf $ADTTMP
-

debian/tests/run-nose3-tests

-#!/bin/sh -e
-
-if [ "$ADTTMP" = "" ] ; then
-  ADTTMP=`mktemp -d /tmp/python3-pysam-test.XXXXXX`
-fi
-cd $ADTTMP
-cp -ra /usr/share/doc/python-pysam/tests/* $ADTTMP
-nosetests3 --nocapture
-cd
-rm -rf $ADTTMP

debian/upstream/metadata

@@ -5,3 +5,4 @@ Registry:
   Entry: NA
 - Name: bio.tools
   Entry: pysam
+Repository: https://github.com/pysam-developers/pysam

devtools/buildwheels.sh

@@ -5,6 +5,10 @@
 #
 # It is best to run this in a fresh clone of the repository!
 #
+# Before running, make sure to update image:
+#
+#   docker pull quay.io/pypa/manylinux1_x86_64
+#
 # Run this within the repository root:
 #   docker run --rm -v $(pwd):/io quay.io/pypa/manylinux1_x86_64 /io/buildwheels.sh
 #
@@ -24,13 +28,6 @@ fi
 
 yum install -y zlib-devel bzip2-devel xz-devel
 
-# Python 2.6 is not supported
-rm -r /opt/python/cp26*
-
-# Python 3.3 builds fail with:
-#  /opt/rh/devtoolset-2/root/usr/libexec/gcc/x86_64-CentOS-linux/4.8.2/ld: cannot find -lchtslib
-rm -r /opt/python/cp33*
-
 # Without libcurl support, htslib can open files from HTTP and FTP URLs.
 # With libcurl support, it also supports HTTPS and S3 URLs, but libcurl needs a
 # current version of OpenSSL, and we do not want to be responsible for