CHANGES.md

 # NCBI External Developer Release:
 
-## NCBI VDB
+## NCBI VDB 2.9.1
+**June 15, 2018**
+
+  **build**: 'make install' ignore ROOT environment variable  
+  **kfg, kns**: Added API to instruct VDB to silently accept any server certificate as valid.  
+  **kfg, vdb-config**: name resolver service now makes use of fcgi  
+  **kfg, vfs**: Fixed a bug that prevented decryption of objects encrypted with non-UTF8 text password keys  
+  **klib**: added check for NULL format argument to string_printf()  
+  **kns**: Randomly select from multiple proxies in configuration  
+  **test**: added Windows test projects for the new schema and the latest vdb  
+
+
+## NCBI VDB 2.9.0
 **February 23, 2018**
 
   **align**: AlignAccessRefSeqEnumeratorNext no longer filters by the index  

build/Makefile.clang

@@ -47,7 +47,7 @@ LP = @ $(TOP)/build/ld.sh $(OS) $(ARCH) clang++ \
 
 # tool options
 WARN = -Wall -Wno-variadic-macros -Wno-long-long # -Wconversion
-
+# TODO: Lots of new clang options in later versions (-fsanitize, CFI, CPI, ...)
 ifeq (64,$(BITS))
 	CARCH = -m64
 else

build/Makefile.env

@@ -330,8 +330,8 @@ DLLX ?= $(SHLX)
 ARCHDEFS = -D_ARCH_BITS=__SIZEOF_POINTER__*__CHAR_BIT__ -DLIBPREFIX=$(LPFX) -DSHLIBEXT=$(DLLX)
 
 # default tool parameters
-CFLAGS	= $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
-CPFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+CFLAGS	= -std=gnu11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+CPFLAGS = -std=gnu++11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
 # some LDFLAGS may have been specified in $(CONFIG_FILE)
 LDFLAGS += $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT)
 

build/Makefile.gcc

@@ -67,19 +67,15 @@ endif
 
 ifeq (dbg, $(BUILD))
 	DBG = -g
-	OPT = $(WARN)
-	NOPT = $(WARN)
+	OPT = $(WARN) -Wno-variadic-macros -fno-strict-aliasing -fstack-protector -Wa,--noexecstack -Wall
+	NOPT = $(WARN) -Wno-variadic-macros -fno-strict-aliasing -fstack-protector -Wa,--noexecstack -Wall
 	# GCC seems unable to use c99 without ansi, which
 	# basically clobbers features back to c89. the options
 	# must be specified in the order shown.
-	PED = -std=gnu99 -pedantic # -fdiagnostics-show-option
+	PED = -std=gnu11 -pedantic # -fdiagnostics-show-option
 else
-
-OPT = -O3 -Wno-variadic-macros -fno-strict-aliasing -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1
-ifeq (x86_64, $(ARCH))
-	OPT += -Wall
-endif
-
+	# TODO: use -fstack-protector-strong for gcc >= 4.9
+	OPT = -O3 -Wno-variadic-macros -fno-strict-aliasing -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -fstack-protector -Wa,--noexecstack -Wall -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-c++11-extensions
 endif
 
 ifeq (mac,$(OS))

build/Makefile.install

@@ -31,6 +31,10 @@ include $(CONFIG_FILE)
 #-------------------------------------------------------------------------------
 # set up installation locations
 
+# unset outside defined variables
+ROOT =
+LINUX_ROOT =
+
 #fake root for debugging
 #uncomment this line and change the following test for root ( see under install: ) to succeed:
 #ROOT = ~/root

build/Makefile.vers

@@ -23,4 +23,4 @@
 # ===========================================================================
 
 # NCBI-VDB and library version
-VERSION = 2.9.0
+VERSION = 2.9.1

debian/changelog

+ncbi-vdb (2.9.1+dfsg-1) UNRELEASED; urgency=medium
+
+  * New upstream version
+  * Point Vcs fields to salsa.debian.org
+  * Standards-Version: 4.1.4
+
+ -- Andreas Tille <tille@debian.org>  Wed, 27 Jun 2018 21:19:14 +0200
+
 ncbi-vdb (2.9.0-1+dfsg-2) unstable; urgency=medium
 
   * Drop some unneeded d-shlibs overrides available in version 0.82

debian/control

@@ -14,9 +14,9 @@ Build-Depends: debhelper (>= 11~),
                libmbedtls-dev,
                zlib1g-dev,
                default-jdk-headless
-Standards-Version: 4.1.3
-Vcs-Browser: https://anonscm.debian.org/cgit/debian-med/ncbi-vdb.git
-Vcs-Git: https://anonscm.debian.org/git/debian-med/ncbi-vdb.git
+Standards-Version: 4.1.4
+Vcs-Browser: https://salsa.debian.org/med-team/ncbi-vdb
+Vcs-Git: https://salsa.debian.org/med-team/ncbi-vdb.git
 Homepage: https://github.com/ncbi/ncbi-vdb
 
 Package: libncbi-vdb2
@@ -170,4 +170,3 @@ Description: sqlite modul of ncbi-vdb library (devel)
  interface.
  .
  This is the development package.
-

debian/patches/mbedtls_ssl_init.patch

@@ -23,7 +23,7 @@ Description: Name change of function probably a consequence of using Debian
                                    &self -> entropy, ( const unsigned char * ) pers, pers_size );
  
      if ( ret != 0 )
-@@ -310,7 +310,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
+@@ -327,7 +327,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
                          /* these guys take a length, so presumably the string is not NUL terminated.
                             yet, the first thing they do is see if the NUL is included in the length! */
                          STATUS ( STAT_GEEK, "Parsing text for node '%s' from CA root certificates\n", cert_name );
@@ -32,7 +32,7 @@ Description: Name change of function probably a consequence of using Debian
                              ( const unsigned char * ) cert_string -> addr, cert_string -> size + 1 );
                      
                          StringWhack ( cert_string );
-@@ -350,7 +350,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
+@@ -374,7 +374,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
              if ( rc2 == 0 )
              {
                  STATUS ( STAT_GEEK, "Parsing text from CA root certificate file '%S'\n", ca_crt_path );
@@ -41,7 +41,7 @@ Description: Name change of function probably a consequence of using Debian
                  if ( ret < 0 )
                  {
                      PLOGMSG ( klogWarn, ( klogWarn
-@@ -371,7 +371,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
+@@ -395,7 +395,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
      if ( num_certs == 0 )
      {
          STATUS ( STAT_QA, "Parsing text for default CA root certificates\n" );
@@ -50,7 +50,7 @@ Description: Name change of function probably a consequence of using Debian
              ( const unsigned char * ) ca_crt_ncbi1, sizeof ca_crt_ncbi1 );
                      
          if ( ret < 0 )
-@@ -388,7 +388,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
+@@ -412,7 +412,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self
          {
              num_certs = 1;
              
@@ -59,7 +59,7 @@ Description: Name change of function probably a consequence of using Debian
                  ( const unsigned char * ) ca_crt_ncbi2, sizeof ca_crt_ncbi2 );
  
              if ( ret >= 0 )
-@@ -416,7 +416,7 @@ rc_t tlsg_setup ( KTLSGlobals * self )
+@@ -440,7 +440,7 @@ rc_t tlsg_setup ( KTLSGlobals * self )
  
      STATUS ( STAT_QA, "Configuring SSl defaults\n" );
  
@@ -68,7 +68,7 @@ Description: Name change of function probably a consequence of using Debian
                                          MBEDTLS_SSL_IS_CLIENT,
                                          MBEDTLS_SSL_TRANSPORT_STREAM,
                                          MBEDTLS_SSL_PRESET_DEFAULT );
-@@ -433,9 +433,9 @@ rc_t tlsg_setup ( KTLSGlobals * self )
+@@ -457,9 +457,9 @@ rc_t tlsg_setup ( KTLSGlobals * self )
          return rc;
      }
  
@@ -81,7 +81,7 @@ Description: Name change of function probably a consequence of using Debian
  
      return 0;
  }
-@@ -470,7 +470,7 @@ static int set_threshold ( const KConfig
+@@ -494,7 +494,7 @@ static int set_threshold ( const KConfig
      }
  
      if ( set )
@@ -90,9 +90,9 @@ Description: Name change of function probably a consequence of using Debian
  
      return ( int ) threshold;
  }
-@@ -481,13 +481,13 @@ rc_t KTLSGlobalsInit ( KTLSGlobals * tls
- {
-     rc_t rc;
+@@ -508,13 +508,13 @@ rc_t KTLSGlobalsInit ( KTLSGlobals * tls
+     assert ( tlsg != NULL );
+     assert ( kfg != NULL );
  
 -    vdb_mbedtls_x509_crt_init ( &tlsg -> cacert );
 -    vdb_mbedtls_ctr_drbg_init ( &tlsg -> ctr_drbg );
@@ -109,10 +109,10 @@ Description: Name change of function probably a consequence of using Debian
  
      rc = tlsg_seed_rng ( tlsg );
      if ( rc == 0 )
-@@ -504,10 +504,10 @@ rc_t KTLSGlobalsInit ( KTLSGlobals * tls
-  */
- void KTLSGlobalsWhack ( KTLSGlobals * self )
+@@ -533,10 +533,10 @@ void KTLSGlobalsWhack ( KTLSGlobals * se
  {
+     assert ( self != NULL );
+     
 -    vdb_mbedtls_ssl_config_free ( &self -> config );
 -    vdb_mbedtls_entropy_free ( &self -> entropy );
 -    vdb_mbedtls_ctr_drbg_free ( &self -> ctr_drbg );
@@ -124,7 +124,7 @@ Description: Name change of function probably a consequence of using Debian
  
      memset ( self, 0, sizeof * self );
  }
-@@ -541,8 +541,8 @@ static
+@@ -609,8 +609,8 @@ static
  void KTLSStreamDestroy ( KTLSStream *self )
  {
      /* tear down all of the stuff created during Make */
@@ -135,7 +135,7 @@ Description: Name change of function probably a consequence of using Debian
  
      /* release the ciphertext object */
      KStreamRelease ( self -> ciphertext );
-@@ -584,7 +584,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
+@@ -652,7 +652,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
      while ( 1 )
      {
          /* read through TLS library */
@@ -144,7 +144,7 @@ Description: Name change of function probably a consequence of using Debian
  
          /* no error */
          if ( ret >= 0 )
-@@ -614,7 +614,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
+@@ -682,7 +682,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
          {
              /* The ret is anything other than the following 3, then the ssl context becomes
               * becomes unusable and should either be freed or call
@@ -153,7 +153,7 @@ Description: Name change of function probably a consequence of using Debian
               * must be closed
               */
          case MBEDTLS_ERR_SSL_WANT_READ: 
-@@ -626,7 +626,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
+@@ -694,7 +694,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
               * is initiating a new connection using the same source port.
               * You can either treat that as a connection close and wait
               * for the client to resend a ClientHello, or directly
@@ -162,7 +162,7 @@ Description: Name change of function probably a consequence of using Debian
               * context (as it has beeen reset internally). Either way, you
               * should make sure this is seen by the application as a new
               * connection: application state, if any, should be reset, and
-@@ -690,7 +690,7 @@ rc_t CC KTLSStreamWrite ( KTLSStream * s
+@@ -758,7 +758,7 @@ rc_t CC KTLSStreamWrite ( KTLSStream * s
          *  We expect to be called through KStreamWriteAll that will
          *  avoid the issue above. 
          */
@@ -171,7 +171,7 @@ Description: Name change of function probably a consequence of using Debian
  
          /* no error */
          if ( ret >= 0 )
-@@ -805,7 +805,7 @@ int CC ktls_net_send ( void *ctx, const
+@@ -873,7 +873,7 @@ int CC ktls_net_send ( void *ctx, const
      return ( int ) num_writ;
  }
  
@@ -180,7 +180,7 @@ Description: Name change of function probably a consequence of using Debian
  static
  int CC ktls_net_recv ( void *ctx, unsigned char *buf, size_t len )
  {
-@@ -850,7 +850,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
+@@ -918,7 +918,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
      assert ( self -> mgr != NULL );
      tlsg = & self -> mgr -> tlsg;
  
@@ -189,7 +189,7 @@ Description: Name change of function probably a consequence of using Debian
      if ( ret != 0 )
      {
          rc_t rc = RC ( rcKrypto, rcSocket, rcFormatting, rcEncryption, rcFailed );
-@@ -891,7 +891,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
+@@ -959,7 +959,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
              return rc;
      }
  
@@ -198,7 +198,7 @@ Description: Name change of function probably a consequence of using Debian
  
      if ( hostz != host )
          StringWhack ( hostz );
-@@ -909,7 +909,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
+@@ -977,7 +977,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
      }
  
  
@@ -207,7 +207,7 @@ Description: Name change of function probably a consequence of using Debian
  
      return 0;
  }
-@@ -921,7 +921,7 @@ rc_t ktls_handshake ( KTLSStream *self )
+@@ -989,7 +989,7 @@ rc_t ktls_handshake ( KTLSStream *self )
  
      STATUS ( STAT_QA, "Performing SSL/TLS handshake...\n" );
  
@@ -216,7 +216,7 @@ Description: Name change of function probably a consequence of using Debian
      while ( ret != 0 )
      {
          if ( ret != MBEDTLS_ERR_SSL_WANT_READ && 
-@@ -938,11 +938,11 @@ rc_t ktls_handshake ( KTLSStream *self )
+@@ -1021,11 +1021,11 @@ rc_t ktls_handshake ( KTLSStream *self )
  
                  if ( ret == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED )
                  {
@@ -230,7 +230,7 @@ Description: Name change of function probably a consequence of using Debian
                          
                          PLOGMSG ( klogSys, ( klogSys
                                               , "mbedtls_ssl_get_verify_result returned $(flags) ( $(info) )"
-@@ -955,7 +955,7 @@ rc_t ktls_handshake ( KTLSStream *self )
+@@ -1039,7 +1039,7 @@ rc_t ktls_handshake ( KTLSStream *self )
  
              return rc;
          }
@@ -239,7 +239,7 @@ Description: Name change of function probably a consequence of using Debian
      }
  
      return 0;
-@@ -991,7 +991,7 @@ rc_t KTLSStreamMake ( KTLSStream ** objp
+@@ -1075,7 +1075,7 @@ rc_t KTLSStreamMake ( KTLSStream ** objp
                      obj -> mgr = mgr;
  
                      STATUS ( STAT_PRG, "%s - initializing tls wrapper\n", __func__ );
@@ -248,9 +248,9 @@ Description: Name change of function probably a consequence of using Debian
  
                      * objp = obj;
                      return 0;
-@@ -1133,13 +1133,13 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACer
+@@ -1217,13 +1217,13 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACer
         rc = RC ( rcKrypto, rcToken, rcValidating, rcSelf, rcNull );
-    else
+    else if ( ! self -> mgr -> tlsg . allow_all_certs )
     {
 -       uint32_t flags = vdb_mbedtls_ssl_get_verify_result( &self -> ssl );
 +       uint32_t flags = mbedtls_ssl_get_verify_result( &self -> ssl );

debian/patches/remove_-m64_flag.patch

@@ -39,10 +39,10 @@ Description: Enable build on x32 by removing explicit -m32/-m64
  endif
 --- a/build/Makefile.clang
 +++ b/build/Makefile.clang
-@@ -48,16 +48,6 @@ LP = @ $(TOP)/build/ld.sh $(OS) $(ARCH)
+@@ -48,15 +48,6 @@ LP = @ $(TOP)/build/ld.sh $(OS) $(ARCH)
  # tool options
  WARN = -Wall -Wno-variadic-macros -Wno-long-long # -Wconversion
- 
+ # TODO: Lots of new clang options in later versions (-fsanitize, CFI, CPI, ...)
 -ifeq (64,$(BITS))
 -	CARCH = -m64
 -else
@@ -52,7 +52,6 @@ Description: Enable build on x32 by removing explicit -m32/-m64
 -	CARCH = -m32
 -endif    
 -endif
--
+ 
  ifeq (prof, $(BUILD))
  	PROF = -pg
- endif

debian/patches/skip_failing_test.patch

@@ -13,7 +13,7 @@ Description: Skip some tests that are failing in pbuilder.
      REQUIRE_RC(KConfigImportNgc(kfg, ngcPath.c_str(), "repos/ngc/", &newRepo));
      TEST_MESSAGE("KConfigImportNgc(" << ngcPath << ")");
      // contents of the input file:
-@@ -978,6 +979,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic
+@@ -979,6 +980,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic
      REQUIRE_EQ(string(newRepo), string("repos/ngc/"));
      
      REQUIRE_RC(KDirectoryRemove(wd, true, "repos"));
@@ -21,7 +21,7 @@ Description: Skip some tests that are failing in pbuilder.
  }
  #endif
  
-@@ -986,15 +988,19 @@ FIXTURE_TEST_CASE(KConfigImportNgc_NullL
+@@ -987,15 +989,19 @@ FIXTURE_TEST_CASE(KConfigImportNgc_NullL
      CreateAndLoad(GetName(), "\n");
      const char* newRepo;
      Cleaner cleaner(wd);
@@ -451,7 +451,7 @@ Description: Skip some tests that are failing in pbuilder.
      }
      
      REQUIRE_RC(VPathRelease(path));
-@@ -213,59 +215,6 @@ protected:
+@@ -214,59 +216,6 @@ protected:
      string m_cachedFile;
  };
  
@@ -513,12 +513,12 @@ Description: Skip some tests that are failing in pbuilder.
      Configure(GetName());
 --- a/test/klib/test-klib.cpp
 +++ b/test/klib/test-klib.cpp
-@@ -772,7 +772,7 @@ TEST_CASE(IsUserAnAdminTest)
-     }
-     else
-     {
+@@ -810,7 +810,7 @@ TEST_CASE(IsUserAnAdminTest)
+     // otherwise, we do not really know
+ #else
+     // Linux or not under TeamCity
 -    REQUIRE(!is_iser_an_admin());
 +    // REQUIRE ( !is_iser_an_admin() ); // makes no sense to check this in Debian Build process
+ #endif
  }
- #else
- #if !defined (MAC)
+ 

debian/patches/use_debian_packaged_libmbedx.patch

@@ -26,7 +26,7 @@ Description: Somehow Build system expects a copy of libmbedx509.a which is
  	$(LD) --slib --vers $(SRCDIR)/libncbi-vdb.vers -o $@ $(VDB_LIB)
 --- a/libs/kns/Makefile
 +++ b/libs/kns/Makefile
-@@ -106,7 +106,8 @@ KNS_OBJ = \
+@@ -107,7 +107,8 @@ KNS_OBJ = \
  
  KNS_LIB = \
      -lkfs \

interfaces/align/samextract-lib.h

+/* ===========================================================================
+ *
+ *                            PUBLIC DOMAIN NOTICE
+ *               National Center for Biotechnology Information
+ *
+ *  This software/database is a "United States Government Work" under the
+ *  terms of the United States Copyright Act.  It was written as part of
+ *  the author's official duties as a United States Government employee and
+ *  thus cannot be copyrighted.  This software/database is freely available
+ *  to the public for use. The National Library of Medicine and the U.S.
+ *  Government have not placed any restriction on its use or reproduction.
+ *
+ *  Although all reasonable efforts have been taken to ensure the accuracy
+ *  and reliability of the software and data, the NLM and the U.S.
+ *  Government do not and cannot warrant the performance or results that
+ *  may be obtained by using this software or data. The NLM and the U.S.
+ *  Government disclaim all warranties, express or implied, including
+ *  warranties of performance, merchantability or fitness for any particular
+ *  purpose.
+ *
+ *  Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#ifndef _h_samextract_lib_
+#define _h_samextract_lib_
+#include <align/extern.h>
+#include <kfs/file.h>
+#include <klib/defs.h>
+#include <klib/rc.h>
+#include <klib/text.h>
+#include <klib/vector.h>
+#include <kproc/queue.h>
+#include <sys/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef int8_t i8;
+typedef uint8_t u8;
+typedef int16_t i16;
+typedef uint16_t u16;
+typedef int32_t i32;
+typedef uint32_t u32;
+typedef int64_t i64;
+typedef uint64_t u64;
+
+typedef enum efile_type {
+    unknown = 999,
+    SAM,
+    BAM,
+    SAMGZUNSUPPORTED
+} efile_type;
+
+typedef struct Alignment
+{
+    const char* qname;
+    uint16_t flags;
+    const char* rname;
+    int32_t pos;
+    uint8_t mapq;
+    const char* cigar;
+    const char* rnext;
+    int32_t pnext;
+    int32_t tlen;
+    const char* read;
+    const char* qual;
+} Alignment;
+
+typedef struct SAMExtractor
+{
+    const KFile* infile;
+
+    Vector headers;
+    Vector alignments;
+    Vector bam_references;
+
+    Vector tagvalues;
+    Vector* prev_headers;
+    Vector* prev_aligns;
+
+    Vector threads;
+    KQueue* inflatequeue;
+    KQueue* parsequeue;
+
+    uint64_t file_pos;
+    char* readbuf;
+    uint32_t readbuf_sz;
+    uint32_t readbuf_pos;
+
+    efile_type file_type;
+
+    rc_t rc;
+
+    String* filter_rname;
+    ssize_t filter_pos;
+    ssize_t filter_length;
+
+    int32_t num_threads;
+    int32_t pos;
+
+    int32_t n_ref;
+
+    bool filter_ordered;
+
+    bool hashdvn;
+    bool hashdso;
+    bool hashdgo;
+    bool hassqsn;
+    bool hassqln;
+    bool hasrgid;
+    bool haspgid;
+} SAMExtractor;
+
+typedef struct tagvalue
+{
+    const char* tag; /* VN, SN, LN, ID, ... */
+    const char* value;
+} TagValue;
+
+typedef struct Header
+{
+    const char* headercode; /* HD, SQ, RG, PG, CO */
+    Vector tagvalues;
+} Header;
+
+ALIGN_EXTERN rc_t CC SAMExtractorMake(SAMExtractor** state, const KFile* fin,
+                                      String* fname_desc,
+                                      int32_t num_threads);
+
+ALIGN_EXTERN rc_t CC SAMExtractorAddFilterName(SAMExtractor* state,
+                                               String* rname, bool ordered);
+ALIGN_EXTERN rc_t CC SAMExtractorAddFilterNamePos(SAMExtractor* state,
+                                                  String* rname, ssize_t pos,
+                                                  bool ordered);
+ALIGN_EXTERN rc_t CC SAMExtractorAddFilterNamePosLength(SAMExtractor* state,
+                                                        String* rname,
+                                                        ssize_t pos,
+                                                        ssize_t length,
+                                                        bool ordered);
+ALIGN_EXTERN rc_t CC SAMExtractorAddFilterPos(SAMExtractor* state,
+                                              ssize_t pos, bool ordered);
+ALIGN_EXTERN rc_t CC SAMExtractorAddFilterPosLength(SAMExtractor* state,
+                                                    ssize_t pos,
+                                                    ssize_t length,
+                                                    bool ordered);
+
+ALIGN_EXTERN rc_t CC SAMExtractorRelease(SAMExtractor* state); /* dtor */
+
+ALIGN_EXTERN rc_t CC SAMExtractorGetHeaders(SAMExtractor* state,
+                                            Vector* headers);
+ALIGN_EXTERN rc_t CC SAMExtractorInvalidateHeaders(SAMExtractor* state);
+
+ALIGN_EXTERN rc_t CC SAMExtractorGetAlignments(SAMExtractor* state,
+                                               Vector* alignments);
+ALIGN_EXTERN rc_t CC SAMExtractorInvalidateAlignments(SAMExtractor* state);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __h_sam_extract_lib_ */

interfaces/cc/gcc/x86_64/arch-impl.h

@@ -134,6 +134,13 @@ int32_t uint32_msbit ( uint32_t self )
     return rtn;
 }
 
+static __inline__
+int32_t uint64_msbit ( uint64_t self )
+{
+    if (self==0) return -1;
+    return 63 - __builtin_clzll ( self );
+}
+
 typedef struct int128_t int128_t;
 struct int128_t
 {
@@ -452,6 +459,7 @@ uint64_t uint64_ror ( uint64_t val, uint8_t bits )
     return rtn;
 }
 
+
 #ifdef __cplusplus
 }
 #endif

interfaces/cc/vc++/noarch/arch-impl.h

@@ -149,6 +149,16 @@ int32_t uint32_msbit ( uint32_t self )
     return uint16_msbit ( ( uint16_t ) self );
 }
 
+#if defined(_WIN64)
+static __inline__
+uint64_t uint64_msbit( uint64_t self )
+{
+    unsigned long idx;
+    _BitScanForward64(&idx, self);
+    return idx;
+}
+#endif
+
 typedef struct int128_t int128_t;
 struct int128_t
 {

interfaces/diagnose/diagnose.h

+#ifndef _h_diagnose_diagnose_
+#define _h_diagnose_diagnose_
+
+/*===========================================================================
+ *
+ *                            PUBLIC DOMAIN NOTICE
+ *               National Center for Biotechnology Information
+ *
+ *  This software/database is a "United States Government Work" under the
+ *  terms of the United States Copyright Act.  It was written as part of
+ *  the author's official duties as a United States Government employee and
+ *  thus cannot be copyrighted.  This software/database is freely available
+ *  to the public for use. The National Library of Medicine and the U.S.
+ *  Government have not placed any restriction on its use or reproduction.
+ *
+ *  Although all reasonable efforts have been taken to ensure the accuracy
+ *  and reliability of the software and data, the NLM and the U.S.
+ *  Government do not and cannot warrant the performance or results that
+ *  may be obtained by using this software or data. The NLM and the U.S.
+ *  Government disclaim all warranties, express or implied, including
+ *  warranties of performance, merchantability or fitness for any particular
+ *  purpose.
+ *
+ *  Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#ifndef _h_diagnose_extern_
+#include <diagnose/extern.h>
+#endif
+
+#ifndef _h_klib_defs_
+#include <klib/defs.h> /* rc_t */
+#endif
+
+#include <stdarg.h> /* va_list */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct KConfig;
+struct KFile;
+struct KNSManager;
+struct VFSManager;
+
+
+typedef struct KDiagnose KDiagnose;
+typedef struct KDiagnoseError KDiagnoseError;
+typedef struct KDiagnoseTest KDiagnoseTest;
+typedef struct KDiagnoseTestDesc KDiagnoseTestDesc;
+
+
+/**************************** Make/AddRef/Release *****************************/
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseMakeExt ( KDiagnose ** test,
+struct KConfig * kfg, struct KNSManager * kmg, struct VFSManager * vmgr, rc_t (CC *quitting)(void));
+DIAGNOSE_EXTERN rc_t CC KDiagnoseAddRef ( const KDiagnose * self );
+DIAGNOSE_EXTERN rc_t CC KDiagnoseRelease ( const KDiagnose * self );
+
+/********************************** Feedback **********************************/
+
+/*
+ * TestHandlerSet: set a test callback
+ * the callback is called when each test is started or finished
+ *
+ * state - state of the test
+ * test - test being executed.
+ *      test remains valid when KDiagnose is valid;
+ *      KDiagnoseTestAddRef if you want to keep it after KDiagnose is released
+ */
+typedef enum {
+    eKDTS_NotStarted,/* test not started yet */
+    eKDTS_Started,   /* test started */
+    eKDTS_Succeed,   /* test finished successfully */
+    eKDTS_Failed,    /* test finished with failure */
+    eKDTS_Skipped,   /* test execution was skipped */
+    eKDTS_Warning,   /* test finished successfully but has a warning for user */
+    eKDTS_Paused,    /* KDiagnosePause was called */
+    eKDTS_Resumed,   /* KDiagnoseResume was called */
+    eKDTS_Canceled,  /* KDiagnoseCancel was called */
+} EKDiagTestState;
+
+/* test is NULL when state is one of:
+ *                                    eKDTS_Paused
+ *                                    eKDTS_Resumed
+ *                                    eKDTS_Canceled
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestHandlerSet ( KDiagnose * self,
+        void ( CC * callback )
+             ( EKDiagTestState state, const KDiagnoseTest * test, void * data ),
+        void * data
+    );
+
+/********************************** Logging **********************************/
+
+#define KVERBOSITY_NONE  -3
+#define KVERBOSITY_ERROR -2
+#define KVERBOSITY_INFO  -1
+#define KVERBOSITY_MAX    0
+DIAGNOSE_EXTERN rc_t CC KDiagnoseSetVerbosity ( KDiagnose * self,
+    int verbosity );
+
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseLogHandlerSet ( KDiagnose * self,
+        rc_t ( CC * logger ) ( int verbosity,
+                            unsigned type, /* TBD */
+                            const char * fmt, va_list args )
+    );
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseLogHandlerSetKOutMsg ( KDiagnose * self );
+
+/********************************* Executing **********************************/
+
+/* All
+ * Default function to execute - diagnose the system.
+ * Check everything and warn when something is not right.
+ * Do not fail when it might not cause failure.
+ * set 'advanced' to 0
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseAll ( KDiagnose * self, uint64_t advanced );
+
+/* Acc
+ * Diagnose user system, make sure 'acc' (optional) can be accessed in the
+ * scope of projectId ( 'projectId' = 0 : public accession ).
+ * Return non-0 rc when it cannot be accessed or downloaded
+ * set 'advanced' to 0
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseAcc ( KDiagnose * self, const char * acc,
+    uint32_t projectId, bool checkHttp, bool checkAspera, bool checkDownload,
+    uint64_t advanced );
+
+/* Kart
+ * Diagnose that the kart file can be accessed.
+ * Check 'numberOfKartItemsToCheck' rows ( 0 means 'all' )
+ * set 'advanced' to 0
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseKart ( KDiagnose * self,
+    const struct KFile * kart, uint32_t numberOfKartItemsToCheck,
+    bool checkHttp, bool checkAspera, uint64_t advanced );
+
+/***************************** Process management *****************************/
+
+DIAGNOSE_EXTERN rc_t CC KDiagnosePause  ( KDiagnose * self );
+DIAGNOSE_EXTERN rc_t CC KDiagnoseResume ( KDiagnose * self );
+DIAGNOSE_EXTERN rc_t CC KDiagnoseCancel ( KDiagnose * self );
+
+/************************ Diagnostics results: errors *************************/
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseGetErrorCount ( const KDiagnose * self,
+    uint32_t * count );
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseGetError ( const KDiagnose * self,
+    uint32_t idx, const KDiagnoseError ** error );
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseErrorAddRef ( const KDiagnoseError * self );
+DIAGNOSE_EXTERN rc_t CC KDiagnoseErrorRelease ( const KDiagnoseError * self );
+
+/* GetMsg:
+ * Get Error Message.
+ * Returned string remains valid while "self" is valid
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseErrorGetMsg ( const KDiagnoseError * self,
+                                               const char ** message );
+
+/******************** Diagnostics results: executed tests *********************/
+
+/* GetTests:
+ * Get executed tests
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseGetTests ( const KDiagnose * self,
+                                            const KDiagnoseTest ** test );
+
+/* Level
+ *
+ * Get test level in tests hierarchy.
+ * 0 is the highest level
+ * tests of 'level 1' are run from test of 'level 0' etc
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestLevel ( const KDiagnoseTest * self,
+                                             uint32_t * level );
+
+/* Number
+ *
+ * Get hiererchical number of test inside of est level in tests hierarchy.
+ * E.g., 0.2.2
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestNumber ( const KDiagnoseTest * self,
+                                              const char ** number );
+
+/* Name
+ *
+ * Get test name.
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestName ( const KDiagnoseTest * self,
+                                            const char ** name );
+
+/* Code
+ *
+ * Get test's code.
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestCode ( const KDiagnoseTest * self,
+                                            uint64_t * code );
+
+/* Message
+ *
+ * Get test message (is set when test is finished)
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestMessage ( const KDiagnoseTest * self,
+                                               const char ** message );
+
+/* State
+ *
+ * Get test state ( changes during test execution )
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestState ( const KDiagnoseTest * self,
+                                             EKDiagTestState * state );
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestNext ( const KDiagnoseTest * self,
+                                            const KDiagnoseTest ** test );
+
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestChild ( const KDiagnoseTest * self, 
+                               uint32_t idx, const KDiagnoseTest ** test );
+
+/********************************** Anvanced **********************************/
+
+/* Advanced
+ * Run KDiagnose tests
+ *
+ * tests is combination of DIAGNOSE_* values
+ *
+ * When "tests | KDIAGN_FAIL != 0" - KDiagnoseAdvanced will return non-0 rc
+ * When KDiagnoseCancel is called - KDiagnose will return rcCanceled
+ *
+ * If 'tests' contain KDIAGN_REPO_GAP
+ * and/or DIAGNOSE_NETWORK_HTTPS/DIAGNOSE_NETWORK_ASPERA
+ * - we will try to check existing dbGaP configuration and access to dbGaP
+ * servers and issue WARNINGS if something is not correct.
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseAdvanced ( KDiagnose * self, uint64_t tests );
+
+#define KDIAGN_FAIL        0x8000000000000000
+
+#define KDIAGN_REPO_REMOTE                0x1
+#define KDIAGN_REPO_SITE                  0x2
+#define KDIAGN_REPO_USER_PUBLIC           0x4
+#define KDIAGN_REPO_GAP                   0x8
+#define KDIAGN_KFG_ASCP                  0x10
+
+#define KDIAGN_ACCESS_NCBI_HTTP          0x20
+#define KDIAGN_ACCESS_NCBI_HTTPS         0x40
+#define KDIAGN_ACCESS_NCBI_FTP           0x80
+#define KDIAGN_ACCESS_NCBI_VERSION      0x100
+
+#define KDIAGN_HTTP                     0x200
+#define KDIAGN_HTTP_RUN               ( 0x400 | KDIAGN_HTTP )
+#define KDIAGN_HTTP_CGI                 0x800
+#define KDIAGN_HTTP_SMALL_ACCESS       0x1000
+#define KDIAGN_HTTP_SMALL_VFS          0x2000
+
+#define KDIAGN_ASCP                    0x4000
+#define KDIAGN_ASCP_RUN              ( 0x8000 | KDIAGN_ASCP )
+#define KDIAGN_ASCP_CGI               0x10000
+#define KDIAGN_ASCP_DOWNLOAD          0x20000
+
+#define KDIAGN_HTTP_VS_ASCP           0x40000
+
+#define KDIAGN_DOWNLOAD_HTTP          0x80000
+#define KDIAGN_DOWNLOAD_ASCP         0x100000
+
+#define KDIAGN_REQUIRE_HTTP_AND_ASCP 0x200000
+
+#define KDIAGN_TRY_TO_WARN           0x400000
+
+#define KDIAGN_AS_IS                 0x800000
+
+#define KDIAGN_KFG_NO_GAP ( KDIAGN_REPO_REMOTE      | KDIAGN_REPO_SITE | \
+                            KDIAGN_REPO_USER_PUBLIC | KDIAGN_KFG_ASCP )
+
+#define KDIAGN_CONFIG ( KDIAGN_KFG_NO_GAP | KDIAGN_REPO_GAP )
+
+#define KDIAGN_ACCESS_NCBI ( KDIAGN_ACCESS_NCBI_HTTP | \
+                          KDIAGN_ACCESS_NCBI_HTTPS  | KDIAGN_ACCESS_NCBI_FTP | \
+                           KDIAGN_ACCESS_NCBI_VERSION)
+
+#define KDIAGN_NETWORK ( KDIAGN_ACCESS_NCBI | \
+              KDIAGN_HTTP | KDIAGN_HTTP_RUN | KDIAGN_HTTP_CGI | \
+              KDIAGN_ASCP | KDIAGN_ASCP_RUN | KDIAGN_ASCP_CGI | \
+              KDIAGN_HTTP_VS_ASCP )
+
+#define KDIAGN_ALL                  ( ~ KDIAGN_FAIL )
+
+/******************************************************************************/
+
+/*#define DIAGNOSE_NETWORK_NCBI    8
+#define DIAGNOSE_NETWORK_HTTPS  16
+#define DIAGNOSE_NETWORK_ASPERA 32
+#define DIAGNOSE_NETWORK_DB_GAP 64
+#define DIAGNOSE_NETWORK \
+    ( DIAGNOSE_NETWORK_NCBI | DIAGNOSE_NETWORK_HTTPS | DIAGNOSE_NETWORK_ASPERA \
+                            | DIAGNOSE_NETWORK_DB_GAP )
+
+#define KDIAGN_ALL ( DIAGNOSE_CONFIG | DIAGNOSE_NETWORK )*/
+
+
+/* DbGap
+ * Diagnose user system, projects with 'projectId' can be accessed.
+ * The last 'projectId' in the argument list is 0
+ *
+ * Configuration WILL BE checked for VALIDIRY of 
+ * projectId's project and access to gbGaP servers.
+ * KDiagnoseDbGap WILL FAIL if configuration is not complete or no gbGaP server
+ * can be accessed.
+ *
+DIAGNOSE_EXTERN rc_t CC KDiagnoseDbGap ( KDiagnose * self, uint64_t tests,
+    uint32_t projectId, ... );*/
+/******************************************************************************/
+
+
+/* GetDesc:
+ * Get description of available tests
+ * Returned object remains valid while "self" is valid
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseGetDesc ( const KDiagnose * self,
+                                           const KDiagnoseTestDesc ** desc );
+
+/* KDiagnoseTestDesc...:
+ * Get description KDiagnoseTestDesc
+ * Returned object remains valid while "self" is valid
+ */
+
+/* Name:
+ * test name
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestDescName ( const KDiagnoseTestDesc * self,
+                                                const char ** name );
+
+/* Desc:
+ * test description
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestDescDesc ( const KDiagnoseTestDesc * self,
+                                                const char ** desc );
+
+/* Desc:
+ * test code to be used to form 'tests' argument of KDiagnoseAdvanced()
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestDescCode ( const KDiagnoseTestDesc * self,
+                                                uint64_t * code );
+
+/* Level:
+ * test level in tests hierarchy.
+ * 0 is the highest level
+ * tests of 'level 1' are subtests of 'level 0' etc
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestDescLevel ( const KDiagnoseTestDesc * self,
+                                                 uint32_t * level );
+
+/* Next:
+ * next test of the same level
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestDescNext ( const KDiagnoseTestDesc * self,
+                                             const KDiagnoseTestDesc ** next );
+
+/* Child:
+ * the first child test of next level
+ */
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestDescChild ( const KDiagnoseTestDesc * self,
+                                             const KDiagnoseTestDesc ** child );
+
+/* Depends:
+ * the test 'self' cannot be executed if test 'depends' fails
+ *
+DIAGNOSE_EXTERN rc_t CC KDiagnoseTestDescDepends (
+    const KDiagnoseTestDesc * self, const KDiagnoseTestDesc * depends );*/
+
+
+#ifdef __cplusplus
+}
+#endif
+
+/******************************************************************************/
+
+#endif /* _h_diagnose_diagnose_ */

interfaces/diagnose/extern.h

+#ifndef _h_diagnose_extern_
+#define _h_diagnose_extern_
+
+/*===========================================================================
+*
+*                            PUBLIC DOMAIN NOTICE
+*               National Center for Biotechnology Information
+*
+*  This software/database is a "United States Government Work" under the
+*  terms of the United States Copyright Act.  It was written as part of
+*  the author's official duties as a United States Government employee and
+*  thus cannot be copyrighted.  This software/database is freely available
+*  to the public for use. The National Library of Medicine and the U.S.
+*  Government have not placed any restriction on its use or reproduction.
+*
+*  Although all reasonable efforts have been taken to ensure the accuracy
+*  and reliability of the software and data, the NLM and the U.S.
+*  Government do not and cannot warrant the performance or results that
+*  may be obtained by using this software or data. The NLM and the U.S.
+*  Government disclaim all warranties, express or implied, including
+*  warranties of performance, merchantability or fitness for any particular
+*  purpose.
+*
+*  Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#if ! defined EXPORT_LATCH && defined _LIBRARY
+#define DIAGNOSE_EXTERN LIB_EXPORT
+#define EXPORT_LATCH 1
+#else
+#define DIAGNOSE_EXTERN LIB_IMPORT
+#endif
+
+
+#ifndef _h_klib_extern_
+#include <klib/extern.h>
+#endif
+
+
+#endif /* _h_diagnose_extern_ */

interfaces/kapp/args.h

@@ -244,6 +244,14 @@ rc_t CC ArgsOptionCount ( const Args * self, const char * option_name, uint32_t
 rc_t CC ArgsOptionValue ( const Args * self, const char * option_name,
     uint32_t iteration, const void ** value );
 
+/* OptionValueExt
+ *  What was the Nth value seen for this option?
+ *  Use OptionCount to know how many were seen.
+ *  Was this option called by long or short name (alias)?
+ */
+rc_t CC ArgsOptionValueExt ( const Args * self, const char * option_name,
+    uint32_t iteration, const void ** value, bool * called_as_alias );
+
 /*
  * ParamCount
  *  How many Parameters were seen?

interfaces/kdb/kdb-priv.h

@@ -85,6 +85,9 @@ KDB_EXTERN rc_t CC KDBManagerVPathOpenLocalDBRead ( struct KDBManager const * se
 KDB_EXTERN rc_t CC KDBManagerVPathOpenRemoteDBRead ( struct KDBManager const * self,
     struct KDatabase const ** db, struct VPath const * remote, struct VPath const * cache );
 
+/** Unreliable object: do not report occured erros */
+KDB_EXTERN int CC KDBManagerVPathTypeUnreliable (
+    const struct KDBManager * self, const char *object, va_list args );
 
 /*--------------------------------------------------------------------------
  * KDatabase

interfaces/kfc/rc.h

@@ -161,6 +161,7 @@ RC_ENUM ( RCTarget )
     RC_ENTRY ( rcData, "data" )
     RC_ENTRY ( rcQuery, "query" )
     RC_ENTRY ( rcUri, "uri" )
+    RC_ENTRY ( rcHashtable, "hashtable" )
     RC_LAST  ( rcLastTarget_v1_1 )
 };