README.md

@@ -130,3 +130,7 @@ ga('send', 'pageview');
   src/filterm4.py 'code'
 [pbdagcon_wf.sh]:
   src/cpp/pbdagcon_wf.sh 'code'
+
+DISCLAIMER
+----------
+THIS WEBSITE AND CONTENT AND ALL SITE-RELATED SERVICES, INCLUDING ANY DATA, ARE PROVIDED "AS IS," WITH ALL FAULTS, WITH NO REPRESENTATIONS OR WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, SATISFACTORY QUALITY, NON-INFRINGEMENT OR FITNESS FOR A PARTICULAR PURPOSE. YOU ASSUME TOTAL RESPONSIBILITY AND RISK FOR YOUR USE OF THIS SITE, ALL SITE-RELATED SERVICES, AND ANY THIRD PARTY WEBSITES OR APPLICATIONS. NO ORAL OR WRITTEN INFORMATION OR ADVICE SHALL CREATE A WARRANTY OF ANY KIND. ANY REFERENCES TO SPECIFIC PRODUCTS OR SERVICES ON THE WEBSITES DO NOT CONSTITUTE OR IMPLY A RECOMMENDATION OR ENDORSEMENT BY PACIFIC BIOSCIENCES.

bamboo_build.sh

+#!/bin/bash
+set -e
+NEXUS_BASEURL=http://ossnexus.pacificbiosciences.com/repository
+#NEXUS_URL=$NEXUS_BASEURL/unsupported/gcc-4.9.2
+if [ ! -e .distfiles/gtest/release-1.7.0.tar.gz ]; then
+  mkdir -p .distfiles/gtest
+  curl -sL $NEXUS_BASEURL/unsupported/distfiles/googletest/release-1.7.0.tar.gz \
+    -o .distfiles/gtest/release-1.7.0.tar.gz
+fi
+tar zxf .distfiles/gtest/release-1.7.0.tar.gz -C repos/
+ln -sfn googletest-release-1.7.0 repos/gtest
+
+rm -rf deployment
+mkdir -p deployment
+/bin/ls -t tarballs/pbbam-*.tgz        | head -1 | xargs -r -n 1 cat | tar zxv --strip-components 3 -C deployment
+/bin/ls -t tarballs/blasr_libcpp-*.tgz | head -1 | xargs -r -n 1 cat | tar zxv --strip-components 3 -C deployment
+/bin/ls -t tarballs/blasr-*.tgz        | head -1 | xargs -r -n 1 cat | tar zxv --strip-components 2 -C deployment
+DEPLOYMENT=$PWD/deployment
+
+export PATH=$PWD/deployment/bin:$PATH
+export LD_LIBRARY_PATH=$PWD/deployment/lib:$LD_LIBRARY_PATH
+
+type module >& /dev/null || . /mnt/software/Modules/current/init/bash
+module load git
+module load gcc
+module load ccache
+if [[ $USER == bamboo ]]; then
+  export CCACHE_DIR=/mnt/secondary/Share/tmp/bamboo.mobs.ccachedir
+fi
+module load boost
+if [[ $BOOST_ROOT =~ /include ]]; then
+  set -x
+  BOOST_ROOT=$(dirname $BOOST_ROOT)
+  set +x
+fi
+module load htslib
+module load hdf5-tools
+module load zlib
+
+DAZZDB=$PWD/repos/dazzdb
+DALIGNER=$PWD/repos/daligner
+cd repos/pbdagcon
+export CCACHE_BASEDIR=$PWD
+rm -rf build
+set -x
+mkdir -p build
+    BOOST_INCLUDE=$BOOST_ROOT/include \
+LIBPBDATA_INCLUDE=$DEPLOYMENT/include/pbdata \
+    LIBPBDATA_LIB=$DEPLOYMENT/lib \
+ LIBBLASR_INCLUDE=$DEPLOYMENT/include/alignment \
+     LIBBLASR_LIB=$DEPLOYMENT/lib \
+LIBPBIHDF_INCLUDE=$DEPLOYMENT/include/hdf \
+    LIBPBIHDF_LIB=$DEPLOYMENT/lib \
+    PBBAM_INCLUDE=$DEPLOYMENT/include \
+        PBBAM_LIB=$DEPLOYMENT/lib \
+    HTSLIB_CFLAGS=$(pkg-config --cflags htslib) \
+      HTSLIB_LIBS=$(pkg-config --libs htslib) \
+     HDF5_INCLUDE=$(pkg-config --cflags-only-I hdf5|awk '{print $1}'|sed -e 's/^-I//') \
+         HDF5_LIB=$(pkg-config --libs-only-L hdf5|awk '{print $1}'|sed -e 's/^-L//') \
+     DALIGNER_SRC=$DALIGNER \
+      DAZZ_DB_SRC=$DAZZDB \
+        GTEST_SRC=$PWD/../gtest/src \
+    GTEST_INCLUDE=$PWD/../gtest/include \
+    ZLIB_LIBFLAGS="$(pkg-config --libs zlib)" \
+./configure.py --build-dir=$PWD/build
+sed -i -e 's/-lpbihdf/-llibcpp/;s/-lblasr//;s/-lpbdata//' build/defines.mk
+make -C build
+cp -a build/src/cpp/pbdagcon $DEPLOYMENT/bin/
+cp -a build/src/cpp/dazcon   $DEPLOYMENT/bin/
+cd ../..
+
+myVERSION=`pbdagcon --version|awk '/version/{print $3}'`
+#rm -rf tarballs && mkdir -p tarballs
+cd deployment
+tar zcf ../tarballs/pbdagcon-${myVERSION}.tgz bin/pbdagcon bin/dazcon

bamboo_test.sh

+#!/bin/bash
+export PATH=$PWD/deployment/bin:$PATH
+export LD_LIBRARY_PATH=$PWD/deployment/lib:$LD_LIBRARY_PATH
+type module >& /dev/null \
+|| . /mnt/software/Modules/current/init/bash
+module load gcc
+module load htslib
+module load hdf5-tools
+module load zlib
+
+make -C repos/pbdagcon/build check
+chmod +w -R repos/pbdagcon

configure.py

@@ -152,7 +152,7 @@ def compose_defines_pacbio(envin):
             'LIBPBIHDF_INCLUDE', 'LIBPBIHDF_LIB', 'LIBPBIHDF_LIBFLAGS',
             'HDF5_INCLUDE', 'HDF5_LIB', 'HDF5_LIBFLAGS',
             'PBBAM_INCLUDE', 'PBBAM_LIB', 'PBBAM_LIBFLAGS',
-            'HTSLIB_INCLUDE', 'HTSLIB_LIB', 'HTSLIB_LIBFLAGS',
+            'HTSLIB_CFLAGS', 'HTSLIB_LIBS',
             'BOOST_INCLUDE', 'PTHREAD_LIBFLAGS',
             'ZLIB_LIB', 'ZLIB_LIBFLAGS',
             'GCC_LIB',
@@ -244,7 +244,7 @@ def set_defs_defaults(env, nopbbam):
     pbbam_defaults = {
         'LIBPBIHDF_LIBFLAGS': '-lpbihdf',
         'PBBAM_LIBFLAGS': '-lpbbam',
-        'HTSLIB_LIBFLAGS': '-lhts',
+        'HTSLIB_LIBS': '-lhts',
         'HDF5_LIBFLAGS': '-lhdf5_cpp -lhdf5',
         'ZLIB_LIBFLAGS': '-lz',
         'PTHREAD_LIBFLAGS': '-lpthread',

debian/changelog

@@ -6,9 +6,12 @@ pbdagcon (0.3+git20180411.c14c422+ds-1) UNRELEASED; urgency=low
   * Add README.source
 
   [ Andreas Tille ]
-  * d/watch: use git mode
-    TODO: Clarify role of code copies pointing to outdated versions of
-          daligner and dazzdb packages
+  * d/watch: use git mode to detect new upstream commit
+  * d/get-orig-source: checkout submodules for code copy of DALIGNER and
+    DAZZ_DB which are specific adaptations to pbdagcon
+  * debhelper 11
+  * Point Vcs fields to salsa.debian.org
+  * Standards-Version: 4.2.0
 
  -- Andreas Tille <tille@debian.org>  Thu, 19 Jul 2018 11:25:43 +0200
 

debian/compat

-10
+11

debian/control

 Source: pbdagcon
+Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
+Uploaders: Afif Elghraoui <afif@debian.org>
 Section: science
 Priority: optional
-Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
-Uploaders:
-	Afif Elghraoui <afif@debian.org>,
-Build-Depends:
-	debhelper (>= 10),
+Build-Depends: debhelper (>= 11~),
                python,
                zlib1g-dev,
                libhdf5-dev,
@@ -14,21 +12,19 @@ Build-Depends:
                libpbseq-dev,
                libpbbam-dev,
 # Test-Depends:
-	googletest,
-Standards-Version: 3.9.8
+               googletest
+Standards-Version: 4.2.0
+Vcs-Browser: https://salsa.debian.org/med-team/pbdagcon
+Vcs-Git: https://salsa.debian.org/med-team/pbdagcon.git
 Homepage: https://github.com/PacificBiosciences/pbdagcon
-Vcs-Git: https://anonscm.debian.org/git/debian-med/pbdagcon.git
-Vcs-Browser: https://anonscm.debian.org/cgit/debian-med/pbdagcon.git
 
 Package: pbdagcon
 Architecture: any
-Depends:
-	${shlibs:Depends},
+Depends: ${shlibs:Depends},
          ${misc:Depends},
          libpbseq,
-	libpbbam,
-Suggests:
-	blasr,
+         libpbbam
+Suggests: blasr
 Description: sequence consensus using directed acyclic graphs
  pbdagcon is a tool that implements DAGCon (Directed Acyclic Graph Consensus)
  which is a sequence consensus algorithm based on using directed acyclic

debian/get-orig-source

@@ -8,28 +8,50 @@ PKGDIR="$PWD"
 # using source package name $SPKG from debian/changelog
 # This makes it more robust for newer uupdate in future
 SPKG=$(dpkg-parsechangelog -SSource)
-NEW_VERSION=$(dpkg-parsechangelog -SVersion | sed 's/\-[0-9]*$//')
+
+#NEW_VERSION=$(dpkg-parsechangelog -SVersion | sed 's/\-[0-9]*$//')
+
+if ! echo $@ | grep -q upstream-version ; then
+    NEW_VERSION=`dpkg-parsechangelog | awk '/^Version:/ { print $2 }' | sed -e 's/+ds[0-9]*//' -e 's/\([0-9\.]\+\)-[0-9]\+$/\1/'`
+    uscan --force-download
+else
+    NEW_VERSION=`echo $@ | sed "s?^.*--upstream-version \([0-9.+gitabcdef]\+\)+ds.*?\1?"`
+    if echo "$VERSION" | grep -q "upstream-version" ; then
+        echo "Unable to parse version number"
+        exit
+    fi
+fi
+
 new_package="${SPKG}-${NEW_VERSION}"
 
 clean () {
-    rm -rf ../${new_package} ../${SPKG}-${NEW_VERSION}.tar.gz
+    rm -rf ../${new_package} ../${SPKG}.tmp.tar.gz
 }
 
 trap clean EXIT
 
 cd ..
-# Now ${SPKG}_${NEW_VERSION}.orig.tar.xz is upstream tarball
-rm -f ${SPKG}_${NEW_VERSION}.orig.tar.*
+# Remove original download since its lacking DALIGNER and DAZZ_DB
+rm -f ${SPKG}-${NEW_VERSION}.tar.[gx]z
 
+# since daligner und dazz_db are needed and are only provided
+# as submodules and this old code seems to be really needed
+#   see https://lists.debian.org/debian-med/2018/07/msg00088.html
 echo "Cloning ${SPKG} version ${NEW_VERSION}"
 git clone \
     https://github.com/PacificBiosciences/pbdagcon.git $new_package
 
-cd $new_package \
-    && make update-submodule \
-    && rm -rf blasr_libcpp \
-    && find -name ".git*" -exec rm -rf {} + \
-    && cd $PKGDIR
-
-tar -czf ../${SPKG}-${NEW_VERSION}.tar.gz ../${new_package}
-mk-origtargz --compression xz ../${SPKG}-${NEW_VERSION}.tar.gz
+cd $new_package
+# Add "|| true" since it returns:
+#  Fetched in submodule path 'blasr_libcpp', but it did not contain 9b4faa3efa2d59484af3c3562a3af6ce421e0294. Direct fetching of that commit failed.
+#  makefile:22: recipe for target 'update-submodule' failed
+make update-submodule || true
+# debug
+# ls -l DALIGNER
+# ls -l DAZZ_DB
+rm -rf blasr_libcpp
+find -name ".git*" | xargs rm -rf
+cd $PKGDIR
+
+tar -czf ../${SPKG}.tmp.tar.gz ../${new_package}
+mk-origtargz --compression xz --repack --version ${NEW_VERSION}+ds ../${SPKG}.tmp.tar.gz

debian/patches/compiler-flags.patch

@@ -2,15 +2,15 @@ Description: Prevent override of dpkg-buildflags
 Author: Afif Elghraoui <afif@debian.org>
 Forwarded: no
 Last-Update: 2016-10-23
---- pbdagcon.orig/src/cpp/makefile
-+++ pbdagcon/src/cpp/makefile
-@@ -8,8 +8,8 @@
+--- a/src/cpp/makefile
++++ b/src/cpp/makefile
+@@ -8,8 +8,8 @@ PBDAGCON_OBJECTS := BlasrM5AlnProvider.o
  DAZCON_OBJECTS := DB.o QV.o align.o DazAlnProvider.o dazcon.o
  
  CPPFLAGS += -MMD -MP
--CXXFLAGS = -O3 -std=c++11 -Wall -Wuninitialized -pedantic
+-CXXFLAGS = -O3 -std=c++14 -Wall -Wuninitialized -pedantic
 -CFLAGS = -O3 -Wall -Wextra -fno-strict-aliasing
-+CXXFLAGS += -O3 -std=c++11 -Wall -Wuninitialized -pedantic
++CXXFLAGS += -O3 -std=c++14 -Wall -Wuninitialized -pedantic
 +CFLAGS += -O3 -Wall -Wextra -fno-strict-aliasing
  
  INCDIRS := \

debian/rules

@@ -11,6 +11,7 @@ PBSEQ_BASE=/usr/include/pbseq
 export LIBBLASR_INCLUDE = $(PBSEQ_BASE)/alignment
 export LIBPBDATA_INCLUDE = $(PBSEQ_BASE)/pbdata
 export LIBPBIHDF_INCLUDE = $(PBSEQ_BASE)/hdf
+export PBBAM_INCLUDE = $(PBSEQ_BASE)
 export HDF5_LIB = /usr/lib/$(DEB_HOST_MULTIARCH)/hdf5/serial
 
 export DALIGNER_SRC = $(CURDIR)/DALIGNER

debian/watch

 version=4
 
 opts="mode=git,gitmode=full,pretty=0.3+git%cd.%h,repacksuffix=+ds,dversionmangle=s/\+ds//g,repack,compression=xz" \
-    https://github.com/PacificBiosciences/pbdagcon.git HEAD
+    https://github.com/PacificBiosciences/pbdagcon.git HEAD \
+    debian debian/get-orig-source

src/cpp/DazAlnProvider.cpp

@@ -222,7 +222,7 @@ int TargetHit::aend() {
 // daligner structures.
 Target::Target(): needsFree_(false) { }
 
-Target::Target(HITS_DB& db, int tspace, int small) :
+Target::Target(DAZZ_DB& db, int tspace, int small) :
     db_(db),
     tspace_(tspace),
     small_(small),

src/cpp/DazAlnProvider.hpp

@@ -94,7 +94,7 @@ std::ostream& operator<<(std::ostream& ostrm, TargetHit& hit);
 class Target {
 public:
     Target();
-    Target(HITS_DB& db, int tspace, int small);
+    Target(DAZZ_DB& db, int tspace, int small);
     ~Target();
 
     // Initializes this target based on the given record, possibly scoring as
@@ -119,7 +119,7 @@ public:
     std::vector<TargetHit> hits;
 
 private:
-    HITS_DB db_;
+    DAZZ_DB db_;
     char* abuffer_, *bbuffer_;
     Work_Data* work_;
     int tspace_, small_;
@@ -158,7 +158,7 @@ private:
     const ProgramOpts popts_;
 
     // Dazzler-related data
-    HITS_DB db_;
+    DAZZ_DB db_;
     int64 novl_, covl_;
     int tbytes_;
     FILE* input_;

src/cpp/makefile

@@ -8,7 +8,7 @@ PBDAGCON_OBJECTS := BlasrM5AlnProvider.o main.o SimpleAligner.o
 DAZCON_OBJECTS := DB.o QV.o align.o DazAlnProvider.o dazcon.o
 
 CPPFLAGS += -MMD -MP
-CXXFLAGS = -O3 -std=c++11 -Wall -Wuninitialized -pedantic
+CXXFLAGS = -O3 -std=c++14 -Wall -Wuninitialized -pedantic
 CFLAGS = -O3 -Wall -Wextra -fno-strict-aliasing
 
 INCDIRS := \
@@ -18,8 +18,7 @@ INCDIRS := \
 	${LIBPBDATA_INCLUDE} \
 	${LIBPBIHDF_INCLUDE} \
 	${PBBAM_INCLUDE} \
-	${HDF5_INCLUDE} \
-	${HTSLIB_INCLUDE}
+	${HDF5_INCLUDE}
 SYS_INCDIRS := \
 	${BOOST_INCLUDE} \
 	${THISDIR}/third-party
@@ -29,7 +28,6 @@ LIBDIRS := \
 	${LIBPBIHDF_LIB} \
 	${PBBAM_LIB} \
 	${HDF5_LIB} \
-	${HTSLIB_LIB} \
 	${GCC_LIB} \
 	${ZLIB_LIB}
 LDLIBS+= \
@@ -38,11 +36,11 @@ LDLIBS+= \
 	${LIBPBIHDF_LIBFLAGS} \
 	${PBBAM_LIBFLAGS} \
 	${HDF5_LIBFLAGS} \
-	${HTSLIB_LIBFLAGS} \
+	${HTSLIB_LIBS} \
 	${ZLIB_LIBFLAGS} \
 	${PTHREAD_LIBFLAGS} \
 	${DL_LIBFLAGS}
-CPPFLAGS+=$(patsubst %,-I%,${INCDIRS}) $(patsubst %,-isystem%,${SYS_INCDIRS})
+CPPFLAGS+=$(patsubst %,-I%,${INCDIRS}) $(patsubst %,-isystem%,${SYS_INCDIRS}) $(HTSLIB_CFLAGS)
 LDFLAGS+=$(patsubst %,-L %,${LIBDIRS})
 LDFLAGS += ${EXTRA_LDFLAGS}
 

src/filterm4.py

@@ -3,6 +3,7 @@
 # Filters for unique, highest scoring subread query/target pairs from an m4
 # file. Helps get rid of chimeras, at the cost of some yield.
 
+from __future__ import print_function
 import sys
 from collections import namedtuple
 
@@ -36,7 +37,7 @@ def printUniq(qgroup, count):
 
     for r in top.values():
         count.filt += 1
-        print r,
+        print(r, end=' ')
 
     qgroup[:] = []
 

src/m4topre.py

@@ -2,6 +2,7 @@
 """Super-simple converter from blasr m4 alignments to pbdagcon 'pre'
 alignments. For use in the pre-assembler dagcon workflow.
 """
+from __future__ import print_function
 
 import sys
 import heapq
@@ -201,8 +202,8 @@ def main(): # pylint: disable=R0914
         else:
             tseq = seqs[rec.tname].translate(__rc__)[::-1][tst:tnd]
 
-        print ' '.join([rec.qname, rec.tname, strand,
-                       rec.tseqlength, str(tst), str(tnd), qseq, tseq])
+        print(' '.join([rec.qname, rec.tname, strand,
+                       rec.tseqlength, str(tst), str(tnd), qseq, tseq]))
 
 if __name__ == '__main__':
     sys.exit(main())

src/q-sense.py

 #!/usr/bin/env python
+from __future__ import print_function
 import sys
 import os
 import logging
@@ -85,8 +86,8 @@ class Consensus(PBMultiToolRunner):
             prefix = ".".join(prefix)
         full_prefix = os.path.join(self.args.out_dir_name, prefix)
         with open("%s_ref.fa" % full_prefix, "w") as f:
-            print >>f ,">%s_ref" % self.args.consensus_seq_name
-            print >>f, s
+            print(">%s_ref" % self.args.consensus_seq_name, file=f)
+            print(s, file=f)
         hp_corr = True if self.args.enable_hp_corr else False
         mark_lower_case = True if self.args.mark_lower_case else False
         generate_consensus(input_fasta_name, "%s_ref.fa" % full_prefix, full_prefix, self.args.consensus_seq_name, 

test/cpp/makefile

@@ -15,7 +15,6 @@ INCDIRS := . \
 	${LIBPBIHDF_INCLUDE} \
 	${PBBAM_INCLUDE} \
 	${HDF5_INCLUDE} \
-	${HTSLIB_INCLUDE} \
 	${BOOST_INCLUDE} \
 	${GTEST_INCLUDE} \
 	third-party
@@ -26,7 +25,6 @@ LIBDIRS := \
 	${LIBPBIHDF_LIB} \
 	${PBBAM_LIB} \
 	${HDF5_LIB} \
-	${HTSLIB_LIB} \
 	${GCC_LIB} \
 	${ZLIB_LIB}
 
@@ -37,11 +35,11 @@ LDLIBS+= \
 	${LIBPBIHDF_LIBFLAGS} \
 	${PBBAM_LIBFLAGS} \
 	${HDF5_LIBFLAGS} \
-	${HTSLIB_LIBFLAGS} \
+	${HTSLIB_LIBS} \
 	${ZLIB_LIBFLAGS} \
 	${PTHREAD_LIBFLAGS} \
 	${DL_LIBFLAGS}
-CPPFLAGS+=$(patsubst %,-I%,${INCDIRS})
+CPPFLAGS+=$(patsubst %,-I%,${INCDIRS}) $(HTSLIB_CFLAGS)
 LDFLAGS+=$(patsubst %,-L %,${LIBDIRS})
 
 # For fused-src, gtest-all.cc includes gtest/*.
@@ -50,7 +48,7 @@ LDFLAGS+=$(patsubst %,-L %,${LIBDIRS})
 GTEST_CPPFLAGS += -isystem $(GTEST_INCLUDE) -isystem ${GTEST_SRC}/..
 GTEST_CXXFLAGS += -g -Wall -Wextra -pthread
 
-CXXFLAGS := -O3 -std=c++11
+CXXFLAGS := -O3 -std=c++14
 
 GTEST_OBJECTS := gtest-all.o gtest_main.o
 DAZCON_OBJECTS := DB.o QV.o align.o Alignment.o \