.gitignore

@@ -33,3 +33,5 @@ user.status
 core
 *.pyc
 .gdb_history
+.vscode
+.gdb_history

CHANGES.md

 # NCBI External Developer Release:
 
+## NCBI VDB
+**February 23, 2018**
+
+  **align**: AlignAccessRefSeqEnumeratorNext no longer filters by the index  
+  **align**: fixed an order-dependent bug affecting cross-table lookups  
+  **build**: Created a script that allows to add a new volume to existing repository  
+  **build**: Fixed configure allowing to run it on Perl with version >= v5.26 that has "." removed from @INC  
+  **build**: added "smoke tests"  
+  **build**: recognize version of libhdf5 that does not allow static linking and do not try to use it  
+  **build, doc**: added wiki page: Building-from-source-:--configure-options-explained  
+  **build, ncbi-vdb, sra-tools**: the installation script now saves configuration files if they were modified by the user  
+  **build, vdb-sql**: modified build to avoid vdb-sql in absence of libxml2  
+  **kfg**: added searching of configuration files in ../etc/ncbi/ relative to the binaries  
+  **kfg, prefetch**: set limit of Aspera usage to 450m  
+  **kfg, prefetch, remote-fuser, vfs**: Updated resolving of cache location of non-accession objects  
+  **kfs**: fix to improve on windows  
+  **klib**: Reverted KTimeMakeTime to use UTC  
+  **kns**: Accept the same http_proxy specifications as wget  
+  **kns**: Added possibility to report server's IP address after network error  
+  **kns**: Ignore HTTP headers sent multiple times  
+  **kns**: Improved reporting of network errors  
+  **kns**: fixed generation of invalid error code in response to dropped connection  
+  **ncbi-vdb**: add ability to make a cache-tee-file without promtion  
+  **ncbi-vdb**: fixed bug of directory not found on mac  
+  **ncbi-vdb, ngs-engine**: improved handling of blobs inside the NGS engine   
+  **ngs-engine**: improved performance when iterating through partially aligned and unaligned reads  
+  **ngs-engine**: optimized filtered access to unaligned runs  
+  **sra-tools, vdb**: access to vdb/ngs via SQLite  
+  **vdb**: An assert triggered by a rare condition was fixed  
+  **vdb**: new api to estimate pileup-workload based on slice  
+  **vdb**: new function to open HTTP-file with an arbitrary page-size  
+  **vdb**: progressbar can now be created to output on stderr  
+  **vfs**: Name resolving service was updated and switched to protocol version 3.0  
+
+
 ## NCBI VDB 2.8.2
 **March 6, 2017**
 

build/Makefile.install

@@ -75,18 +75,12 @@ copyincludes: checkversion
 	@cp $(TOP)/interfaces/ngs/ncbi/NGS.hpp $(INST_INCDIR)/ncbi-vdb
 
 copykfg: checkversion
-	@ #
-	@ # install configuration file(s)
 	@ echo "Installing configuration files to $(KONFIG_DIR)"
-	@ mkdir -p $(KONFIG_DIR)
-	@ if [ -f $(KONFIG_DIR)/certs.kfg   -a "$$(diff $(TOP)/libs/kfg/certs.kfg   $(KONFIG_DIR)/certs.kfg 2>/dev/null)" != "" ] ; \
-        then mv -v $(KONFIG_DIR)/certs.kfg    $(KONFIG_DIR)/certs.kfg.orig    ; fi
-	@ cp $(TOP)/libs/kfg/certs.kfg   $(KONFIG_DIR)
-	@ if [ -f $(KONFIG_DIR)/default.kfg -a "$$(diff $(TOP)/libs/kfg/default.kfg $(KONFIG_DIR)/default.kfg 2>/dev/null)" != "" ] ; \
-        then mv -v $(KONFIG_DIR)/default.kfg  $(KONFIG_DIR)/default.kfg.orig  ; fi
+	@ $(TOP)/build/install-kfg.sh default.kfg $(TOP)/libs/kfg $(KONFIG_DIR) $(TOP)/libs/kfg/kfgsums
+	@ $(TOP)/build/install-kfg.sh certs.kfg   $(TOP)/libs/kfg $(KONFIG_DIR) $(TOP)/libs/kfg/kfgsums
+	@ # rename obsolete ncbi-vdb.kfg
 	@ if [ -f $(KONFIG_DIR)/ncbi-vdb.kfg ] ; \
         then mv -v $(KONFIG_DIR)/ncbi-vdb.kfg  $(KONFIG_DIR)/ncbi-vdb.kfg.orig  ; fi
-	@ cp $(TOP)/libs/kfg/default.kfg $(KONFIG_DIR)
 
 install: copylibs copyincludes copykfg
 ifeq (true, $(LINUX_ROOT))

build/Makefile.vers

@@ -23,4 +23,4 @@
 # ===========================================================================
 
 # NCBI-VDB and library version
-VERSION = 2.8.2
+VERSION = 2.9.0

test/kfg/config.schema.kfg → build/install-kfg.sh

+#!/bin/bash
 # ===========================================================================
 #
 #                            PUBLIC DOMAIN NOTICE
@@ -22,4 +23,30 @@
 #
 # ===========================================================================
 
-vdb/schema/paths = "/panfs/traces01/trace_software/sdk/interfaces"
+# install-kfg.sh
+#   copies file $1 from $2 to $3.
+#   Will create a backup copy if the file's md5 is not found in $4 (assumed to be a user's edit)
+#
+
+FILE_NAME=$1
+SRC_DIR=$2
+KONFIG_DIR=$3
+MD5SUMS=$4
+
+SRC_FILE=$2/$1
+TGT_FILE=$3/$1
+
+mkdir -p ${KONFIG_DIR}
+
+# create a backup if installed file has been modified by user
+if [ -f ${TGT_FILE} ] ; then
+    md5=$(md5sum ${TGT_FILE} | awk '{print $1;}')
+    # echo "$1 md5=$md5"
+    if [ "$(grep ${md5} ${MD5SUMS})" == "" ] ; then
+        # not a known version of the file; create a backup copy
+        mv -v ${TGT_FILE} ${TGT_FILE}.orig
+    fi
+fi
+
+# copy to the install location
+cp ${SRC_FILE} ${TGT_FILE}

build/ld.mac.slib.sh

@@ -51,8 +51,28 @@ else
     CMD="$CMD $OUTDIR/$NAME$DBGAP.a.$VERS"
 fi
 
+# create alternate names for object files
+mkdir -p ld-tmp
+
+ALT_OBJS=""
+for f in $OBJS
+do
+    d="$(dirname $f)"
+    f="$(basename $f)"
+    if [ "$d" != "${d#/}" ]
+    then
+#        echo ln -sf $d/$f ld-tmp/$NAME.$f
+        ln -sf $d/$f ld-tmp/$NAME.$f
+    else
+#        echo ln -sf ../$d/$f ld-tmp/$NAME.$f
+        ln -sf ../$d/$f ld-tmp/$NAME.$f
+    fi
+#    echo ALT_OBJS="ALT_OBJS ld-tmp/$NAME.$f"
+    ALT_OBJS="$ALT_OBJS ld-tmp/$NAME.$f"
+done
+
 # tack on object files
-CMD="$CMD $OBJS"
+CMD="$CMD $ALT_OBJS"
 
 # initial dependency upon Makefile and vers file
 DEPS="$SRCDIR/Makefile"

build/valgrind.suppress

@@ -19,3 +19,26 @@
    ...
    fun:_ZN16ChildProcessFail8Instance11test_methodEv
 }
+{
+   ktst_exception_in_child_process
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:malloc
+   fun:__cxa_allocate_exception
+   ...
+   fun:_ZN16ChildProcessFail8Instance11test_methodEv
+}
+
+{
+   bison_error_recovery_leaks_whitespace_in_AST_parser
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:realloc
+   fun:AddWhitespace
+   fun:schema_yylex
+   fun:SchemaScan_yylex
+   fun:_Z12Schema_parsePPN4ncbi12SchemaParser9ParseTreeEPNS0_11ErrorReportEP15SchemaScanBlock
+   fun:_ZN4ncbi12SchemaParser12SchemaParser11ParseStringEPKcb
+   fun:_ZN11AST_Fixture7MakeAstEPKc
+   fun:_ZN24Const_UnterminatedString8Instance11test_methodEv
+}
\ No newline at end of file

configure

@@ -41,5 +41,5 @@ else
         echo >&2 "configure: error: perl not found."; exit 1; }
 
     cd $CURDIR/setup
-    perl -w konfigure.perl "$@"
+    perl  konfigure.perl "$@"
 fi

debian/changelog

+ncbi-vdb (2.9.0-1+dfsg-1) unstable; urgency=medium
+
+  * New upstream version
+  * Standards-Version: 4.1.3
+  * debhelper 11
+  * d/rules: s/DEB_BUILD_OPTIONS/DH_BUILD_MAINT_OPTIONS/
+
+ -- Andreas Tille <tille@debian.org>  Mon, 19 Mar 2018 11:36:13 +0100
+
 ncbi-vdb (2.8.2-2+dfsg-1) unstable; urgency=medium
 
   * New upstream version

debian/compat

-10
+11

debian/control

@@ -3,7 +3,7 @@ Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.
 Uploaders: Andreas Tille <tille@debian.org>
 Section: science
 Priority: optional
-Build-Depends: debhelper (>= 10),
+Build-Depends: debhelper (>= 11~),
                d-shlibs,
                libbz2-dev,
                libxml2-dev,
@@ -14,7 +14,7 @@ Build-Depends: debhelper (>= 10),
                libmbedtls-dev,
                zlib1g-dev,
                default-jdk-headless
-Standards-Version: 4.1.1
+Standards-Version: 4.1.3
 Vcs-Browser: https://anonscm.debian.org/cgit/debian-med/ncbi-vdb.git
 Vcs-Git: https://anonscm.debian.org/git/debian-med/ncbi-vdb.git
 Homepage: https://github.com/ncbi/ncbi-vdb

debian/patches/debug_configure.patch

@@ -5,7 +5,7 @@ Description: Force debugging output (--with-debug in d/rules is ignored
 
 --- a/setup/konfigure.perl
 +++ b/setup/konfigure.perl
-@@ -435,6 +435,7 @@ if ($TOOLS =~ /gcc$/) {
+@@ -448,6 +448,7 @@ if ( $PKG{REQ} ) {
  
  my @dependencies;
  

debian/patches/find_hdf5_serial.patch

@@ -6,13 +6,13 @@ Description: use hdf5 serial
 
 --- a/setup/konfigure.perl
 +++ b/setup/konfigure.perl
-@@ -1488,8 +1488,8 @@ sub check_compiler {
+@@ -1523,8 +1523,8 @@ sub check_compiler {
              $flags = $n;
              $log = '                      int main() {                     }\n'
          } elsif ($n eq 'hdf5') {
--            $library = '-lhdf5';
+-            $library = '-Wl,-Bstatic -lhdf5 -Wl,-Bdynamic -ldl -lm -lz';
 -            $log = '#include <hdf5.h>  \n int main() { H5close         (); }\n'
-+            $library = '-lhdf5_serial';
++            $library = '-lhdf5_serial -Wl,-Bdynamic -ldl -lm -lz';
 +            $log = '#include <hdf5/serial/hdf5.h>  \n int main() { H5close         (); }\n'
          } elsif ($n eq 'fuse') {
              $flags = '-D_FILE_OFFSET_BITS=64';

debian/patches/find_ngs-sdk.patch

@@ -4,7 +4,7 @@ Description: Enable finding ngs-sdk
 
 --- a/setup/konfigure.perl
 +++ b/setup/konfigure.perl
-@@ -1313,7 +1313,9 @@ sub find_in_dir {
+@@ -1332,7 +1332,9 @@ sub find_in_dir {
          print "\tlibraries... " unless ($AUTORUN);
          if ($lib) {
              my $builddir = File::Spec->catdir($dir, $OS, $TOOLS, $ARCH, $BUILD);
@@ -15,7 +15,7 @@ Description: Enable finding ngs-sdk
              my $ilibdir = File::Spec->catdir($builddir, 'ilib');
              my $f = File::Spec->catdir($libdir, $lib);
              print "\n\t\tchecking $f\n\t" if ($OPT{'debug'});
-@@ -1394,8 +1396,8 @@ sub find_in_dir {
+@@ -1413,8 +1415,8 @@ sub find_in_dir {
          }
      }
      if ($jar) {
@@ -28,7 +28,7 @@ Description: Enable finding ngs-sdk
              $found_lib = $try;
 --- a/setup/package.prl
 +++ b/setup/package.prl
-@@ -30,8 +30,8 @@ sub REQ { ( { name    => 'ngs-sdk',
+@@ -31,8 +31,8 @@ sub REQ { ( { name    => 'ngs-sdk',
                option  => 'with-ngs-java-prefix',
                origin  => 'I',
                type    => 'JO',

debian/patches/mbedtls_ssl_init.patch

@@ -85,10 +85,10 @@ Description: Name change of function probably a consequence of using Debian
      }
  
      if ( set )
--        vdb_mbedtls_debug_set_threshold ( threshold );
-+        mbedtls_debug_set_threshold ( threshold );
+-        vdb_mbedtls_debug_set_threshold ( ( int ) threshold );
++        mbedtls_debug_set_threshold ( ( int ) threshold );
  
-     return threshold;
+     return ( int ) threshold;
  }
 @@ -481,13 +481,13 @@ rc_t KTLSGlobalsInit ( KTLSGlobals * tls
  {
@@ -248,7 +248,7 @@ Description: Name change of function probably a consequence of using Debian
  
                      * objp = obj;
                      return 0;
-@@ -1114,13 +1114,13 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACer
+@@ -1133,13 +1133,13 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACer
         rc = RC ( rcKrypto, rcToken, rcValidating, rcSelf, rcNull );
     else
     {

debian/patches/skip_failing_test.patch

@@ -3,87 +3,9 @@ Last-Update: Tue, 12 Jul 2016 16:15:32 +0200
 Description: Skip some tests that are failing in pbuilder.
  FIXME: There are more tests to skip but I have given up for the moment.
 
---- a/test/kfs/cacheteetest.cpp
-+++ b/test/kfs/cacheteetest.cpp
-@@ -482,7 +482,7 @@ TEST_CASE( CacheTee_Multiple_Users_Singl
- #if !defined(WINDOWS) && !defined(_WIN32)
- TEST_CASE( CacheTee_ReadOnly )
- {
--	KOutMsg( "Test: CacheTee_ReadOnly\n" );
-+	KOutMsg( "***** FIXME: Parts of Test are skipped: CacheTee_ReadOnly *****\n" );
- 	remove_file( CACHEFILE );	// to start with a clean slate on caching...
- 	remove_file( CACHEFILE1 );
- 
-@@ -508,7 +508,7 @@ TEST_CASE( CacheTee_ReadOnly )
- 	/* we read all from the tee-file that should have promoted it on Release,
- 	   but we made it read only before the creation of the 2nd tee-file
- 	   because of that it should not be promoted and not complete */
--	
-+/*
- 	const KFile * cache;
- 	REQUIRE_RC_FAIL( KDirectoryOpenFileRead( dir, &cache, "%s", CACHEFILE ) );
- 	REQUIRE_RC( KDirectoryOpenFileRead( dir, &cache, "%s", CACHEFILE1 ) );
-@@ -520,13 +520,13 @@ TEST_CASE( CacheTee_ReadOnly )
- 	REQUIRE_RC( KFileRelease( cache ) );		
- 	REQUIRE_RC( KFileRelease( org ) );	
- 	REQUIRE_RC( KDirectoryRelease( dir ) );
--  
-+*/ 
- }
- #endif
- 
- TEST_CASE( CacheTee_Multiple_Users_with_Promoting )
- {
--	KOutMsg( "Test: CacheTee_Multiple_Users_with_Promoting\n" );
-+	KOutMsg( "***** FIXME: Parts of Test are skipped: CacheTee_Multiple_Users_with_Promoting *****\n" );
- 	remove_file( CACHEFILE );	// to start with a clean slate on caching...
- 	remove_file( CACHEFILE1 );
- 
-@@ -542,24 +542,26 @@ TEST_CASE( CacheTee_Multiple_Users_with_
- 	const KFile * tee2;
- 	REQUIRE_RC( KDirectoryMakeCacheTee ( dir, &tee2, org, BLOCKSIZE, "%s", CACHEFILE ) );
- 
--	/* read all from tee1 and release it, that will trigger promotion */
-+/*
-+	/ * read all from tee1 and release it, that will trigger promotion * /
- 	REQUIRE_RC( read_all( tee1, 1024 * 32 )	);
- 	REQUIRE_RC( KFileRelease( tee1 ) );
- 
--	/* read a little bit from tee2 and release it, will it corrupt the cache? */	
-+	/ * read a little bit from tee2 and release it, will it corrupt the cache? * /	
- 	REQUIRE_RC( read_partial( tee2, 100, 100 ) );
- 	REQUIRE_RC( KFileRelease( tee2 ) );
- 
--	/* the ( newly ) promoted cache file has to be not corrupt */
-+	/ * the ( newly ) promoted cache file has to be not corrupt * /
- 	REQUIRE_RC( KDirectoryMakeCacheTee ( dir, &tee1, org, BLOCKSIZE, "%s", CACHEFILE ) );
- 	REQUIRE_RC( KFileRelease( tee1 ) );
- 
--	/* the .cache - file has to be gone */
-+	/ * the .cache - file has to be gone * /
- 	uint32_t pt = KDirectoryPathType ( dir, "%s", CACHEFILE1 );
- 	REQUIRE( pt == kptNotFound );
- 	
- 	REQUIRE_RC( KFileRelease( org ) );	
- 	REQUIRE_RC( KDirectoryRelease( dir ) );
-+*/
- }
- 
- //////////////////////////////////////////// Main
---- a/test/klib/test-klib.cpp
-+++ b/test/klib/test-klib.cpp
-@@ -776,7 +776,7 @@ TEST_CASE(IsUserAnAdminTest)
-     }
- #else
-     // Linux or not under TeamCity
--    REQUIRE ( !is_iser_an_admin() );
-+    // REQUIRE ( !is_iser_an_admin() ); // makes no sense to check this in Debian Build process
- #endif
- }
- 
 --- a/test/kfg/kfgtest.cpp
 +++ b/test/kfg/kfgtest.cpp
-@@ -907,6 +907,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic
+@@ -913,6 +913,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic
      string ngcPath("./prj_2956.ngc");
      C::t(ngcPath);
      Cleaner cleaner(wd);
@@ -91,7 +13,7 @@ Description: Skip some tests that are failing in pbuilder.
      REQUIRE_RC(KConfigImportNgc(kfg, ngcPath.c_str(), "repos/ngc/", &newRepo));
      TEST_MESSAGE("KConfigImportNgc(" << ngcPath << ")");
      // contents of the input file:
-@@ -972,6 +973,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic
+@@ -978,6 +979,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic
      REQUIRE_EQ(string(newRepo), string("repos/ngc/"));
      
      REQUIRE_RC(KDirectoryRemove(wd, true, "repos"));
@@ -99,7 +21,7 @@ Description: Skip some tests that are failing in pbuilder.
  }
  #endif
  
-@@ -980,15 +982,19 @@ FIXTURE_TEST_CASE(KConfigImportNgc_NullL
+@@ -986,15 +988,19 @@ FIXTURE_TEST_CASE(KConfigImportNgc_NullL
      CreateAndLoad(GetName(), "\n");
      const char* newRepo;
      Cleaner cleaner(wd);
@@ -589,29 +511,14 @@ Description: Skip some tests that are failing in pbuilder.
  FIXTURE_TEST_CASE(KDBManagerVPathOpenDB_Remote_BadPath, RemoteDBFixture)
  {
      Configure(GetName());
-@@ -280,25 +229,3 @@ FIXTURE_TEST_CASE(KDBManagerVPathOpenDB_
-     REQUIRE_RC(KDBManagerRelease(mgr));
+--- a/test/klib/test-klib.cpp
++++ b/test/klib/test-klib.cpp
+@@ -772,7 +772,7 @@ TEST_CASE(IsUserAnAdminTest)
      }
- 
--FIXTURE_TEST_CASE(KDBManagerVPathOpenDB_Remote_BadCache, RemoteDBFixture)
--{
--    Configure(GetName());
--    Resolve("SRR600096");
--    
--    KDBManager* mgr;
--    REQUIRE_RC(KDB_MANAGER_MAKE(&mgr, NULL));
--    const KDatabase * db;
--    
--    VPath* cache;
--    REQUIRE_RC(VFSManagerMakePath(m_vfsmgr, &cache, "/dev/null")); // unlikely to be usable
--    REQUIRE_RC(KDBManagerVPathOpenRemoteDBRead ( mgr, &db, m_path, cache )); 
--    REQUIRE_RC(VPathRelease(cache));
--
--    /* read something */
--    const KTable *tbl;
--    REQUIRE_RC(KDatabaseOpenTableRead(db, &tbl, "SEQUENCE")); // works anyway, just not caching
--    REQUIRE_RC(KTableRelease(tbl));
--    
--    REQUIRE_RC(KDatabaseRelease(db));
--    REQUIRE_RC(KDBManagerRelease(mgr));
--}
+     else
+     {
+-        REQUIRE ( !is_iser_an_admin() );
++        // REQUIRE ( !is_iser_an_admin() ); // makes no sense to check this in Debian Build process
+     }
+ #else
+ #if !defined (MAC)

debian/patches/spelling.patch

@@ -213,7 +213,7 @@ Description: Fix some spelling issues
  
 --- a/libs/kns/linux/sysendpoint.c
 +++ b/libs/kns/linux/sysendpoint.c
-@@ -131,10 +131,10 @@ rc_t CC KNSManagerInitDNSEndpoint ( stru
+@@ -141,10 +141,10 @@ rc_t CC KNSManagerInitDNSEndpoint ( stru
                          rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
                          break;
  #endif

debian/patches/use_debian_packaged_libmbedx.patch

@@ -26,7 +26,7 @@ Description: Somehow Build system expects a copy of libmbedx509.a which is
  	$(LD) --slib --vers $(SRCDIR)/libncbi-vdb.vers -o $@ $(VDB_LIB)
 --- a/libs/kns/Makefile
 +++ b/libs/kns/Makefile
-@@ -105,7 +105,8 @@ KNS_OBJ = \
+@@ -106,7 +106,8 @@ KNS_OBJ = \
  
  KNS_LIB = \
      -lkfs \
@@ -49,7 +49,7 @@ Description: Somehow Build system expects a copy of libmbedx509.a which is
  #include <mbedtls/aes.h>
 --- a/libs/kns/tls.c
 +++ b/libs/kns/tls.c
-@@ -58,7 +58,7 @@ struct KTLSStream;
+@@ -59,7 +59,7 @@ struct KTLSStream;
  #include MBEDTLS_CONFIG_FILE 
  #endif
  

debian/patches/use_debian_packaged_zlib.patch

@@ -8,7 +8,7 @@ Description: External copy of bz2 and zlib are removed - exclude these from
 @@ -35,7 +35,6 @@ include $(TOP)/build/Makefile.config
  # default
  #
- SUBDIRS =    \
+ MAIN_SUBDIRS = \
 -	ext      \
  	kfc      \
  	klib     \

debian/rules

@@ -2,7 +2,7 @@
 
 export DEB_BUILD_MAINT_OPTIONS=hardening=+all
 
-DEB_BUILD_OPTIONS := nocheck
+DH_BUILD_MAINT_OPTIONS := nocheck
 
 # DH_VERBOSE := 1