CHANGES.md

 # NCBI External Developer Release:
 
+
+## NCBI VDB 2.9.2-1
+**July 31, 2018**
+  **vdb**: restored lost exported attribute on several entries in the private API
+
+
+## NCBI VDB 2.9.2
+**July 23, 2018**
+  **kfg, vfs**: Introduced enhanced handling of download-only NGC files that lack read/decrypt permissions
+
+
 ## NCBI VDB 2.9.1-1
 **June 25, 2018**
 

build/Makefile.vers

@@ -23,4 +23,4 @@
 # ===========================================================================
 
 # NCBI-VDB and library version
-VERSION = 2.9.1
+VERSION = 2.9.2

debian/changelog

+ncbi-vdb (2.9.2-1+dfsg-1) UNRELEASED; urgency=medium
+
+  * New upstream version
+
+  [ Steve Langasek ]
+  * add missing implementation of uint64_msbit for i386
+    Closes: #907358
+
+  [ Andreas Tille ]
+  * d/rules:
+     - DEB_LDFLAGS_MAINT_APPEND := -Wl,-z,defs -Wl,--as-needed
+       This should prevent successful build of a broken library (thanks
+       for the hint to Steve Langasek in #907358)
+     - dh_auto_build --no-parallel
+  * Standards-Version: 4.2.1
+  TODO: Try to fix linking but failed
+
+ -- Andreas Tille <tille@debian.org>  Tue, 28 Aug 2018 07:55:59 +0200
+
 ncbi-vdb (2.9.1-1+dfsg-1) unstable; urgency=medium
 
   * New upstream version

debian/control

@@ -14,7 +14,7 @@ Build-Depends: debhelper (>= 11~),
                libmbedtls-dev,
                zlib1g-dev,
                default-jdk-headless
-Standards-Version: 4.1.4
+Standards-Version: 4.2.1
 Vcs-Browser: https://salsa.debian.org/med-team/ncbi-vdb
 Vcs-Git: https://salsa.debian.org/med-team/ncbi-vdb.git
 Homepage: https://github.com/ncbi/ncbi-vdb

debian/patches/fix_linking.patch

+--- a/libs/hdf5/Makefile
++++ b/libs/hdf5/Makefile
+@@ -112,7 +112,9 @@ ifdef HDF5_LIBDIR
+ endif
+ 
+ $(LIBDIR)/libkdf5.$(SHLX): $(KDF5_OBJ)
+-	$(LD) --dlib --vers $(SRCDIR)/../ncbi-vdb/libncbi-vdb.vers -o $@ $^ $(KDF5_LIB) -lmbedx509 -lmbedtls -lmbedcrypto
++	$(LD) --dlib --vers $(SRCDIR)/../ncbi-vdb/libncbi-vdb.vers -o $@ $^ $(KDF5_LIB) -lmbedx509 -lmbedtls -lmbedcrypto -lklib -lkfs
++	# This needs to be added at the *end* of the linker command line after ... -ldl -lpthread -lm
++	# -lkq -lkproc -lkfs -lkdb -lkns -lkfg
+ 
+ $(LIBDIR)/libkdf5.$(LIBX): $(KDF5_OBJ)
+ 	$(LD) --slib --vers $(SRCDIR)/../ncbi-vdb/libncbi-vdb.vers -o $@ $^ $(KDF5_LIB)
+--- a/libs/ncbi-vdb/Makefile
++++ b/libs/ncbi-vdb/Makefile
+@@ -124,7 +124,7 @@ VDB_LIB = \
+ 
+ $(LIBDIR)/libncbi-vdb.$(SHLX): $(VDB_OBJ)
+ 	echo $(SRCDIR)
+-	$(LD) --dlib --vers $(SRCDIR)/libncbi-vdb.vers -o $@ $(VDB_LIB) -lmbedx509 -lmbedtls -lmbedcrypto
++	$(LD) --dlib --vers $(SRCDIR)/libncbi-vdb.vers -o $@ $(VDB_LIB) -lmbedx509 -lmbedtls -lmbedcrypto -lkq -lkproc -lkfs -lkns -lkfg -lkproc -lksproc
+ 
+ $(LIBDIR)/libncbi-vdb.$(LIBX): $(VDB_OBJ)
+ 	$(LD) --slib --vers $(SRCDIR)/libncbi-vdb.vers -o $@ $(VDB_LIB)
+--- a/build/ld.linux.dlib.sh
++++ b/build/ld.linux.dlib.sh
+@@ -286,6 +292,12 @@ then
+     CMD="$CMD -lm"
+ fi
+ 
++# kdf5 lib needs extra libraries at the **end** of the linker line
++if pwd | grep -q "libs/hdf5$" ; then
++    echo "Try to add extra libs to LDFLAGS ... unfortunately this fails as well"
++    echo CMD="$CMD -lkproc -lkfs -lkdb -lkns -lkfg"
++fi
++
+ # produce shared library
+ echo "$CMD"
+ $CMD || exit $?

debian/patches/i386-uint64_msbit.patch

+Description: add missing implementation of uint64_msbit for i386
+ Without this definition, we get a "successful" library build with undefined
+ symbols on i386.  We can use the same implementation of this function as on
+ x86_64.
+Author: Steve Langasek <steve.langasek@ubuntu.com>
+Last-Modified: 2018-08-26
+
+Index: ncbi-vdb-2.9.1-1+dfsg/interfaces/cc/gcc/i386/arch-impl.h
+===================================================================
+--- ncbi-vdb-2.9.1-1+dfsg.orig/interfaces/cc/gcc/i386/arch-impl.h
++++ ncbi-vdb-2.9.1-1+dfsg/interfaces/cc/gcc/i386/arch-impl.h
+@@ -127,6 +127,13 @@ int32_t uint32_msbit ( uint32_t self )
+     return rtn;
+ }
+ 
++static __inline__
++int32_t uint64_msbit ( uint64_t self )
++{
++    if (self==0) return -1;
++    return 63 - __builtin_clzll ( self );
++}
++
+ typedef struct int128_t int128_t;
+ struct int128_t
+ {

debian/patches/series

@@ -9,3 +9,5 @@ remove_-m64_flag.patch
 add_-msse2.patch
 use_debian_packaged_libmbedx.patch
 mbedtls_ssl_init.patch
+i386-uint64_msbit.patch
+fix_linking.patch

debian/rules

@@ -18,6 +18,8 @@ LIBINSTALLDIR := debian/tmp/usr/$(DEB_SOURCE)/$(OS)/gcc/$(VDB_ARCH)/dbg/lib
 LIBPKG := libncbi-vdb2
 SCHEMADIR := debian/$(LIBPKG)/usr/lib/$(DEB_SOURCE)
 
+export DEB_LDFLAGS_MAINT_APPEND := -Wl,-z,defs -Wl,--as-needed
+
 %:
 	dh $@
 
@@ -38,6 +40,9 @@ override_dh_auto_configure:
 	# no idea how to convince configure script to use the correct include dir so patching result afterwards
 	sed -i 's?HDF5_INCDIR = /usr/include?&/hdf5/serial?' build/Makefile.config*
 
+override_dh_auto_build:
+	dh_auto_build --no-parallel
+
 override_dh_auto_test:
 ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS)))
 	# FIXME: some tests keep on failing despite debian/patches/skip_failing_test.patch - needs to be sorted out later

libs/blast/blast-mgr.c

@@ -50,7 +50,7 @@
 #include <stdio.h> /* fprintf */
 #include <string.h> /* memset */
 
-#define TOOLKIT "sratoolkit2_9_1"
+#define TOOLKIT "sratoolkit2_9_2"
 
 /******************************************************************************/
 

libs/kdb/libkdb.vers.h

@@ -24,4 +24,4 @@
 *
 */
 
-#define LIBKDB_VERS 0x0207001A
+#define LIBKDB_VERS 0x0207001B

libs/kfg/keystore.c

@@ -174,6 +174,11 @@ rc_t ReadEncKey(const KFile* file, char* buf, size_t bufsize)
 
     if (rc == 0)
     {
+        char * pc;
+
+        /* ensure buffer is NUL terminated */
+        buf [ readNum ] = 0;
+        
         /* -----
          * trim back the contents of the file to
          * a single ASCII/UTF-8 text line
@@ -181,17 +186,20 @@ rc_t ReadEncKey(const KFile* file, char* buf, size_t bufsize)
          * end of line characters so it could have other
          * control characters...
          */
-        char* pc = string_chr (buf, readNum, '\r');
+        pc = memchr ( buf, '\r', readNum );
         if ( pc == NULL )
-            pc = string_chr (buf, readNum, '\n');
-
+            pc = memchr ( buf, '\n', readNum );
         if (pc != NULL)
             *pc = 0;
-        else
-            buf[readNum]=0;
 
-        if (string_measure(buf, NULL) == 0)
+        /* disallow a length of 0 */
+        if ( buf [ 0 ] == 0 )
             rc = RC (rcKFG, rcEncryptionKey, rcRetrieving, rcSize, rcTooShort);
+        else if ( memcmp ( buf, "n/a", 4 ) == 0 )
+        {
+            /* download-only NGC file */
+            rc = RC ( rcKFG, rcEncryptionKey, rcRetrieving, rcEncryptionKey, rcNoPerm );
+        }
     }
     return rc;
 }

libs/klib/release-vers.h

@@ -28,7 +28,7 @@
 
 
 /* Version of current SRA Toolkit Release */
-#define RELEASE_VERS 0x02090001
+#define RELEASE_VERS 0x02090002
 
 
 /* Type of Version of current SRA Toolkit Release is one of:

libs/ncbi-vdb/libncbi-vdb.vers

-2.9.1
+2.9.2

libs/vdb/cursor-cmn.c

@@ -167,7 +167,7 @@ bool VCursorIsReadOnly ( const struct VCursor * self )
 {
     DISPATCH_VALUE ( isReadOnly ( self ), true );
 }
-struct VSchema const * VCursorGetSchema ( struct VCursor const *self )
+LIB_EXPORT struct VSchema const * VCursorGetSchema(struct VCursor const *self)
 {
     DISPATCH_VALUE ( getSchema ( self ), NULL );
 }
@@ -183,27 +183,27 @@ const struct KSymbol * VCursorFindOverride ( const VCursor *self, const struct V
 {
     DISPATCH_VALUE ( findOverride ( self, cid ), NULL );
 }
-rc_t CC VCursorPermitPostOpenAdd ( struct VCursor const *self )
+LIB_EXPORT rc_t CC VCursorPermitPostOpenAdd(struct VCursor const *self)
 {
     DISPATCH ( permitPostOpenAdd ( self ) );
 }
-rc_t CC VCursorSuspendTriggers ( struct VCursor const *self )
+LIB_EXPORT rc_t CC VCursorSuspendTriggers(struct VCursor const *self)
 {
     DISPATCH ( suspendTriggers ( self ) );
 }
-rc_t CC VCursorLinkedCursorGet( const struct VCursor * self,const char * tbl, struct VCursor const ** curs)
+LIB_EXPORT rc_t CC VCursorLinkedCursorGet(const struct VCursor * self, const char * tbl, struct VCursor const ** curs)
 {
     DISPATCH ( linkedCursorGet ( self, tbl, curs ) );
 }
-rc_t CC VCursorLinkedCursorSet( const struct VCursor * self,const char * tbl, struct VCursor const * curs)
+LIB_EXPORT rc_t CC VCursorLinkedCursorSet(const struct VCursor * self, const char * tbl, struct VCursor const * curs)
 {
     DISPATCH ( linkedCursorSet ( self, tbl, curs ) );
 }
-uint64_t CC VCursorSetCacheCapacity ( struct VCursor * self,uint64_t capacity )
+LIB_EXPORT uint64_t CC VCursorSetCacheCapacity(struct VCursor * self, uint64_t capacity)
 {
     DISPATCH_VALUE ( setCacheCapacity ( self, capacity ), 0 );
 }
-uint64_t CC VCursorGetCacheCapacity ( const struct VCursor * self )
+LIB_EXPORT uint64_t CC VCursorGetCacheCapacity(const struct VCursor * self)
 {
     DISPATCH_VALUE ( getCacheCapacity ( self ), 0 );
 }

libs/vdb/libvdb.vers.h

@@ -24,4 +24,4 @@
 *
 */
 
-#define LIBVDB_VERS 0x0207001A
+#define LIBVDB_VERS 0x0207001B

libs/vfs/manager.c

@@ -677,6 +677,14 @@ rc_t GetEncryptionKey(const VFSManager * self, const VPath * vpath, char* obuff,
         }
     }
     
+    if ( GetRCState ( rc ) == rcNoPerm && GetRCObject ( rc ) == ( enum RCObject ) rcEncryptionKey )
+    {
+        LOGMSG ( klogErr, "You do not have read permissions to decrypt data from this project." );
+        LOGMSG ( klogErr, "Please contact your PI to request an NGC token with decrypt permissions." );
+        LOGMSG ( klogErr, "Import the new NGC file before decrypting again." );
+        LOGMSG ( klogErr, "If you continue to have problems, contact sra@ncbi.nlm.nih.gov for assistance." );
+    }
+    
     rc2 = KKeyStoreSetTemporaryKeyFromFile(self->keystore, NULL); /* forget the temp key if set */
     if (rc == 0)
         rc = rc2;

test/vdb/test-vdb.cpp

@@ -211,7 +211,7 @@ public:
             throw logic_error ( "~VdbFixture: VCursorRelease failed" );
     }
 
-    rc_t Setup( const char * acc, const char* column[] )
+    rc_t Setup( const char * acc, const char* column[], bool open = true )
     {
         const VDatabase *db = NULL;
         rc_t rc = VDBManagerOpenDBRead ( mgr, &db, NULL, acc );
@@ -255,7 +255,7 @@ public:
                     assert(i < N);
                     rc = VCursorAddColumn ( curs, col_idx + i, column[i] );
                 }
-                if ( rc == 0 )
+                if ( rc == 0 && open )
                 {
                     rc = VCursorOpen(curs);
                 }
@@ -368,7 +368,6 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_SRA, VdbFixture)
 {   // multiple fragments per row (some are technical), multiple rows per blob
     static char const *columns[] = { "READ", 0 };
     REQUIRE_RC ( Setup ( "SRR000123", columns ) );
-    REQUIRE_RC ( VCursorOpen (curs ) );
 
     {
         REQUIRE_RC ( VCursorSetRowId (curs, 1 ) );
@@ -439,7 +438,6 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_WGS, VdbFixture)
 {   // single fragment per row, multiple rows per blob
     static char const *columns[] = { "READ", 0 };
     REQUIRE_RC ( Setup ( "ALWZ01", columns ) );
-    REQUIRE_RC ( VCursorOpen (curs ) );
 
     {
         REQUIRE_RC ( VCursorSetRowId (curs, 1 ) );
@@ -510,7 +508,6 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_SequentialAccess, VdbFixture)
 {   // VDB-2858: sequential access to blobs broken
     static char const *columns[] = { "READ", 0 };
     REQUIRE_RC ( Setup ( "ALAI01", columns ) );
-    REQUIRE_RC ( VCursorOpen (curs ) );
 
     int64_t first;
     uint64_t count;
@@ -543,7 +540,6 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_RandomAccess, VdbFixture)
 {
     static char const *columns[] = { "READ", 0 };
     REQUIRE_RC ( Setup ( "SRR000001", columns ) );
-    REQUIRE_RC ( VCursorOpen (curs ) );
 
     // when accessing randomly, blob sizes stay very small
     REQUIRE ( CheckBlobRange ( 1, 1, 4 ) );
@@ -561,7 +557,6 @@ FIXTURE_TEST_CASE(PageMapIterator_WGS, VdbFixture)
 {   // single fragment per row, multiple rows per blob
     static char const *columns[] = { "READ", 0 };
     REQUIRE_RC ( Setup ( "ALWZ01", columns ) );
-    REQUIRE_RC ( VCursorOpen (curs ) );
 
     {
         REQUIRE_RC ( VCursorSetRowId (curs, 1 ) );
@@ -602,7 +597,6 @@ FIXTURE_TEST_CASE ( VCursor_FindNextRowIdDirect, VdbFixture )
 {
     static char const *columns[] = { "SPOT_ID", "READ", 0 };
     REQUIRE_RC ( Setup ( "SRR000001", columns ) );
-    REQUIRE_RC ( VCursorOpen (curs ) );
     int64_t next;
     REQUIRE_RC ( VCursorFindNextRowIdDirect ( curs, 0, 1, & next ) );
     REQUIRE_EQ ( (int64_t)1, next ) ;
@@ -610,6 +604,13 @@ FIXTURE_TEST_CASE ( VCursor_FindNextRowIdDirect, VdbFixture )
     REQUIRE_EQ ( (int64_t)2, next ) ; // VDB-3075: next == 1
 }
 
+FIXTURE_TEST_CASE(VCursor_PermitPostOpenAdd, VdbFixture)
+{
+    static char const *columns[] = { "SPOT_ID", 0 };
+    REQUIRE_RC(Setup("SRR000001", columns, false));
+    REQUIRE_RC(VCursorPermitPostOpenAdd(curs));
+}
+
 //////////////////////////////////////////// Main
 extern "C"
 {